kevin/tacitus
Archived
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix validator logic

Browse Source
This commit is contained in:
Kevin Cotugno 2018-01-13 01:04:44 -08:00
parent 622a26eb00
commit 4e36400f3e
3 changed files with 34 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
gdax/public_client.go
View File

@ -4,16 +4,24 @@ import (
"github.com/kcotugno/tacitus"
"encoding/json"
"errors"
"net/http"
"sort"
"strconv"
"strings"
"time"
)
type byTradeId []tacitus.Trade
const (
endpoint = "https://api.gdax.com"
)
func (t byTradeId) Len() int { return len(t) }
func (t byTradeId) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
func (t byTradeId) Less(i, j int) bool { return t[i].TradeId < t[j].TradeId }
type PublicClient struct {
client http.Client
}
@ -27,7 +35,7 @@ func NewPublicClient() *PublicClient {
func (c *PublicClient) GetTradesBefore(product string, id int) ([]tacitus.Trade, error) {
url := strings.Join([]string{endpoint, "products", product, "trades"}, "/")
url = strings.Join([]string{url, "?after=", strconv.Itoa(id)}, "")
url = strings.Join([]string{url, "?after=", strconv.Itoa(id + 100)}, "")
resp, err := c.client.Get(url)
if err != nil {
@ -35,6 +43,10 @@ func (c *PublicClient) GetTradesBefore(product string, id int) ([]tacitus.Trade,
}
defer resp.Body.Close()
if resp.StatusCode == 429 {
return nil, errors.New("Rate limit exceeded")
}
results := make([]tradeResponse, 0)
decoder := json.NewDecoder(resp.Body)
@ -62,5 +74,6 @@ func (c *PublicClient) GetTradesBefore(product string, id int) ([]tacitus.Trade,
trades = append(trades, trade)
}
sort.Sort(byTradeId(trades))
return trades, nil
}

30
ops/validator.go
View File

@ -4,16 +4,9 @@ import (
"github.com/kcotugno/tacitus"
"github.com/kcotugno/tacitus/gdax"
"sort"
"time"
)
type byTradeId []tacitus.Trade
func (t byTradeId) Len() int { return len(t) }
func (t byTradeId) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
func (t byTradeId) Less(i, j int) bool { return t[i].TradeId < t[j].TradeId }
type Validator struct {
Database tacitus.DatabaseClientService
Logger tacitus.Logger
@ -111,16 +104,23 @@ func (v *Validator) getMissingTrades(product string, groups [][]int) {
total := 1 + group[1] - group[0]
v.Logger.Info("validator: retrieving %v missing trade(s): %v", total, group)
for i := group[1]; i >= group[0]; i-- {
ts, _ := c.GetTradesBefore(product, i+1)
sort.Sort(byTradeId(ts))
for i := group[0]; i <= group[1]; i += 100 {
ts, err := c.GetTradesBefore(product, i)
if err != nil {
if err.Error() == "Rate limit exceeded" {
v.Logger.Info("validator: Rate limit exceeded, sleeping...")
time.Sleep(1 * time.Second)
i -= 100
continue
} else {
continue
}
}
for _, t := range ts {
if t.TradeId < group[0] && t.TradeId > group[1] {
i = ts[len(ts)-1].TradeId
} else {
if t.TradeId >= group[0] && t.TradeId <= group[1] {
v.Database.TradeService().CreateTrade(t)
i = t.TradeId
}
}
}
@ -135,7 +135,7 @@ func (v *Validator) findMissingGroups(product string, starting int) ([][]int, in
var trade tacitus.Trade
missing := [][]int{}
current := starting
current := starting - 1
for results.Next() {
trade = results.Value().(tacitus.Trade)

10
postgres/trade_service.go
View File

@ -9,9 +9,9 @@ import (
)
const (
trade_columns = ` trade_id, product, price, size, buy, sell, timestamp `
trade_columns = `trade_id, product, price, size, buy, sell, timestamp`
trade_insert = `INSERT INTO trades (` + trade_columns + `) VALUES ` +
`($1, $2, $3, $4, $5, $6, $7) RETURNING id,` + trade_columns + `;`
`($1, $2, $3, $4, $5, $6, $7) RETURNING id, ` + trade_columns + `;`
trade_find = `SELECT id, ` + trade_columns + ` FROM trades WHERE id = $1;`
trade_find_trade_id = `SELECT id, ` + trade_columns + ` FROM trades ` +
`WHERE trade_id = $1 AND product = $2;`
@ -27,9 +27,9 @@ const (
trade_before = `SELECT id, ` + trade_columns + ` FROM trades ` +
`WHERE product = $1 AND trade_id > $2 ORDER BY trade_id ASC LIMIT $3;`
trade_product = `SELECT DISTINCT product FROM trades;`
trade_after_all = `SELECT id,` + trade_columns + `FROM trades WHERE product = $1 ` +
`AND trade_id >= $2 ORDER BY trade_id ASC;`
trade_where = `SELECT id,` + trade_columns + `FROM trades WHERE`
trade_after_all = `SELECT id, ` + trade_columns + ` FROM trades WHERE product = $1 ` +
`AND trade_id > $2 ORDER BY trade_id ASC;`
trade_where = `SELECT id, ` + trade_columns + ` FROM trades WHERE`
)
type TradeService struct {