Compare commits
6 Commits
master
...
fix-valida
Author | SHA1 | Date | |
---|---|---|---|
4a6e7a71bb | |||
4e36400f3e | |||
622a26eb00 | |||
21b93e315e | |||
8ef848ddee | |||
e635f8659e |
@ -7,10 +7,13 @@ import (
|
||||
)
|
||||
|
||||
type Aggregation struct {
|
||||
Id int `json:"id"`
|
||||
Interval int `json:"interval"`
|
||||
Product string `json:"product"`
|
||||
Price decimal.Decimal `json:"price"`
|
||||
Id int `json:"id"`
|
||||
Interval int `json:"interval"`
|
||||
Product string `json:"product"`
|
||||
OpenningPrice decimal.Decimal
|
||||
Price decimal.Decimal
|
||||
LowPrice decimal.Decimal
|
||||
HighPrice decimal.Decimal
|
||||
BuyVolume decimal.Decimal `json:"buy_volume"`
|
||||
SellVolume decimal.Decimal `json:"sell_volume"`
|
||||
BuyTransactions int `json:"buy_transactions"`
|
||||
|
@ -4,16 +4,24 @@ import (
|
||||
"github.com/kcotugno/tacitus"
|
||||
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type byTradeId []tacitus.Trade
|
||||
|
||||
const (
|
||||
endpoint = "https://api.gdax.com"
|
||||
)
|
||||
|
||||
func (t byTradeId) Len() int { return len(t) }
|
||||
func (t byTradeId) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
|
||||
func (t byTradeId) Less(i, j int) bool { return t[i].TradeId < t[j].TradeId }
|
||||
|
||||
type PublicClient struct {
|
||||
client http.Client
|
||||
}
|
||||
@ -27,7 +35,7 @@ func NewPublicClient() *PublicClient {
|
||||
|
||||
func (c *PublicClient) GetTradesBefore(product string, id int) ([]tacitus.Trade, error) {
|
||||
url := strings.Join([]string{endpoint, "products", product, "trades"}, "/")
|
||||
url = strings.Join([]string{url, "?after=", strconv.Itoa(id)}, "")
|
||||
url = strings.Join([]string{url, "?after=", strconv.Itoa(id + 100)}, "")
|
||||
|
||||
resp, err := c.client.Get(url)
|
||||
if err != nil {
|
||||
@ -35,6 +43,10 @@ func (c *PublicClient) GetTradesBefore(product string, id int) ([]tacitus.Trade,
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == 429 {
|
||||
return nil, errors.New("Rate limit exceeded")
|
||||
}
|
||||
|
||||
results := make([]tradeResponse, 0)
|
||||
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
@ -62,5 +74,6 @@ func (c *PublicClient) GetTradesBefore(product string, id int) ([]tacitus.Trade,
|
||||
trades = append(trades, trade)
|
||||
}
|
||||
|
||||
sort.Sort(byTradeId(trades))
|
||||
return trades, nil
|
||||
}
|
||||
|
@ -4,16 +4,9 @@ import (
|
||||
"github.com/kcotugno/tacitus"
|
||||
"github.com/kcotugno/tacitus/gdax"
|
||||
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type byTradeId []tacitus.Trade
|
||||
|
||||
func (t byTradeId) Len() int { return len(t) }
|
||||
func (t byTradeId) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
|
||||
func (t byTradeId) Less(i, j int) bool { return t[i].TradeId > t[j].TradeId }
|
||||
|
||||
type Validator struct {
|
||||
Database tacitus.DatabaseClientService
|
||||
Logger tacitus.Logger
|
||||
@ -111,21 +104,26 @@ func (v *Validator) getMissingTrades(product string, groups [][]int) {
|
||||
total := 1 + group[1] - group[0]
|
||||
v.Logger.Info("validator: retrieving %v missing trade(s): %v", total, group)
|
||||
|
||||
for i := group[1]; i >= group[0]; i-- {
|
||||
ts, _ := c.GetTradesBefore(product, i+1)
|
||||
sort.Sort(byTradeId(ts))
|
||||
for i := group[0]; i <= group[1]; i += 100 {
|
||||
ts, err := c.GetTradesBefore(product, i)
|
||||
if err != nil {
|
||||
if err.Error() == "Rate limit exceeded" {
|
||||
v.Logger.Info("validator: Rate limit exceeded, sleeping...")
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
i -= 100
|
||||
continue
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
for _, t := range ts {
|
||||
if t.TradeId < group[0] && t.TradeId > group[1] {
|
||||
i = ts[len(ts)-1].TradeId
|
||||
} else {
|
||||
if t.TradeId >= group[0] && t.TradeId <= group[1] {
|
||||
v.Database.TradeService().CreateTrade(t)
|
||||
i = t.TradeId
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
v.Logger.Info("validator: DONE")
|
||||
}
|
||||
}
|
||||
|
||||
@ -137,7 +135,7 @@ func (v *Validator) findMissingGroups(product string, starting int) ([][]int, in
|
||||
|
||||
var trade tacitus.Trade
|
||||
missing := [][]int{}
|
||||
current := starting
|
||||
current := starting - 1
|
||||
|
||||
for results.Next() {
|
||||
trade = results.Value().(tacitus.Trade)
|
||||
|
@ -9,15 +9,16 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
trade_columns = ` trade_id, product, price, size, buy, sell, timestamp `
|
||||
trade_columns = `trade_id, product, price, size, buy, sell, timestamp`
|
||||
trade_insert = `INSERT INTO trades (` + trade_columns + `) VALUES ` +
|
||||
`($1, $2, $3, $4, $5, $6, $7) RETURNING id,` + trade_columns + `;`
|
||||
`($1, $2, $3, $4, $5, $6, $7) RETURNING id, ` + trade_columns + `;`
|
||||
trade_find = `SELECT id, ` + trade_columns + ` FROM trades WHERE id = $1;`
|
||||
trade_find_trade_id = `SELECT id, ` + trade_columns + ` FROM trades ` +
|
||||
`WHERE trade_id = $1 AND product = $2;`
|
||||
trade_delete = `DELETE FROM trades WHERE id = $1;`
|
||||
trade_in_date_range = `SELECT id, ` + trade_columns + ` FROM ` +
|
||||
`trades WHERE product = $1 AND timestamp >= $2 AND timestamp < $3;`
|
||||
`trades WHERE product = $1 AND timestamp >= $2 AND timestamp < $3 ` +
|
||||
`ORDER BY timestamp ASC;`
|
||||
trade_first = `SELECT id, ` + trade_columns + ` FROM trades WHERE ` +
|
||||
`product = $1 ORDER BY trade_id ASC LIMIT $2;`
|
||||
trade_last = `SELECT id, ` + trade_columns + ` FROM trades WHERE ` +
|
||||
@ -27,9 +28,9 @@ const (
|
||||
trade_before = `SELECT id, ` + trade_columns + ` FROM trades ` +
|
||||
`WHERE product = $1 AND trade_id > $2 ORDER BY trade_id ASC LIMIT $3;`
|
||||
trade_product = `SELECT DISTINCT product FROM trades;`
|
||||
trade_after_all = `SELECT id,` + trade_columns + `FROM trades WHERE product = $1 ` +
|
||||
`AND trade_id >= $2 ORDER BY trade_id ASC;`
|
||||
trade_where = `SELECT id,` + trade_columns + `FROM trades WHERE`
|
||||
trade_after_all = `SELECT id, ` + trade_columns + ` FROM trades WHERE product = $1 ` +
|
||||
`AND trade_id > $2 ORDER BY trade_id ASC;`
|
||||
trade_where = `SELECT id, ` + trade_columns + ` FROM trades WHERE `
|
||||
)
|
||||
|
||||
type TradeService struct {
|
||||
|
Reference in New Issue
Block a user