Fix validator logic

gdax/public_client.go

@@ -4,16 +4,24 @@ import (
 	"github.com/kcotugno/tacitus"
 
 	"encoding/json"
+	"errors"
 	"net/http"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 )
 
+type byTradeId []tacitus.Trade
+
 const (
 	endpoint = "https://api.gdax.com"
 )
 
+func (t byTradeId) Len() int           { return len(t) }
+func (t byTradeId) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
+func (t byTradeId) Less(i, j int) bool { return t[i].TradeId < t[j].TradeId }
+
 type PublicClient struct {
 	client http.Client
 }
@@ -27,7 +35,7 @@ func NewPublicClient() *PublicClient {
 
 func (c *PublicClient) GetTradesBefore(product string, id int) ([]tacitus.Trade, error) {
 	url := strings.Join([]string{endpoint, "products", product, "trades"}, "/")
-	url = strings.Join([]string{url, "?after=", strconv.Itoa(id)}, "")
+	url = strings.Join([]string{url, "?after=", strconv.Itoa(id + 100)}, "")
 
 	resp, err := c.client.Get(url)
 	if err != nil {
@@ -35,6 +43,10 @@ func (c *PublicClient) GetTradesBefore(product string, id int) ([]tacitus.Trade,
 	}
 	defer resp.Body.Close()
 
+	if resp.StatusCode == 429 {
+		return nil, errors.New("Rate limit exceeded")
+	}
+
 	results := make([]tradeResponse, 0)
 
 	decoder := json.NewDecoder(resp.Body)
@@ -62,5 +74,6 @@ func (c *PublicClient) GetTradesBefore(product string, id int) ([]tacitus.Trade,
 		trades = append(trades, trade)
 	}
 
+	sort.Sort(byTradeId(trades))
 	return trades, nil
 }

ops/validator.go

@@ -4,16 +4,9 @@ import (
 	"github.com/kcotugno/tacitus"
 	"github.com/kcotugno/tacitus/gdax"
 
-	"sort"
 	"time"
 )
 
-type byTradeId []tacitus.Trade
-
-func (t byTradeId) Len() int           { return len(t) }
-func (t byTradeId) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
-func (t byTradeId) Less(i, j int) bool { return t[i].TradeId < t[j].TradeId }
-
 type Validator struct {
 	Database tacitus.DatabaseClientService
 	Logger   tacitus.Logger
@@ -111,16 +104,23 @@ func (v *Validator) getMissingTrades(product string, groups [][]int) {
 		total := 1 + group[1] - group[0]
 		v.Logger.Info("validator: retrieving %v missing trade(s): %v", total, group)
 
-		for i := group[1]; i >= group[0]; i-- {
+		for i := group[0]; i <= group[1]; i += 100 {
-			ts, _ := c.GetTradesBefore(product, i+1)
+			ts, err := c.GetTradesBefore(product, i)
-			sort.Sort(byTradeId(ts))
+			if err != nil {
+				if err.Error() == "Rate limit exceeded" {
+					v.Logger.Info("validator: Rate limit exceeded, sleeping...")
+
+					time.Sleep(1 * time.Second)
+					i -= 100
+					continue
+				} else {
+					continue
+				}
+			}
 
 			for _, t := range ts {
-				if t.TradeId < group[0] && t.TradeId > group[1] {
+				if t.TradeId >= group[0] && t.TradeId <= group[1] {
-					i = ts[len(ts)-1].TradeId
-				} else {
 					v.Database.TradeService().CreateTrade(t)
-					i = t.TradeId
 				}
 			}
 		}
@@ -135,7 +135,7 @@ func (v *Validator) findMissingGroups(product string, starting int) ([][]int, in
 
 	var trade tacitus.Trade
 	missing := [][]int{}
-	current := starting
+	current := starting - 1
 
 	for results.Next() {
 		trade = results.Value().(tacitus.Trade)

postgres/trade_service.go

@@ -9,9 +9,9 @@ import (
 )
 
 const (
-	trade_columns = ` trade_id, product, price, size, buy, sell, timestamp `
+	trade_columns = `trade_id, product, price, size, buy, sell, timestamp`
 	trade_insert  = `INSERT INTO trades (` + trade_columns + `) VALUES ` +
-		`($1, $2, $3, $4, $5, $6, $7) RETURNING id,` + trade_columns + `;`
+		`($1, $2, $3, $4, $5, $6, $7) RETURNING id, ` + trade_columns + `;`
 	trade_find          = `SELECT id, ` + trade_columns + ` FROM trades WHERE id = $1;`
 	trade_find_trade_id = `SELECT id, ` + trade_columns + ` FROM trades ` +
 		`WHERE trade_id = $1 AND product = $2;`
@@ -27,9 +27,9 @@ const (
 	trade_before = `SELECT id, ` + trade_columns + ` FROM trades ` +
 		`WHERE product = $1 AND trade_id > $2 ORDER BY trade_id ASC LIMIT $3;`
 	trade_product   = `SELECT DISTINCT product FROM trades;`
-	trade_after_all = `SELECT id,` + trade_columns + `FROM trades WHERE product = $1 ` +
+	trade_after_all = `SELECT id, ` + trade_columns + ` FROM trades WHERE product = $1 ` +
-		`AND trade_id >= $2 ORDER BY trade_id ASC;`
+		`AND trade_id > $2 ORDER BY trade_id ASC;`
-	trade_where = `SELECT id,` + trade_columns + `FROM trades WHERE`
+	trade_where = `SELECT id, ` + trade_columns + ` FROM trades WHERE`
 )
 
 type TradeService struct {