Merge pull request #1024 from c9s/fix/binance-my-trades-api

fix: binance my trades api

.github/workflows/go.yml

@@ -27,7 +27,7 @@ jobs:
 
     - uses: actions/checkout@v2
       with:
-        lfs: 'true'
+        # lfs: 'true'
         ssh-key: ${{ secrets.git_ssh_key }}
 
     - uses: actions/cache@v2

go.mod

@@ -7,7 +7,7 @@ go 1.17
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/Masterminds/squirrel v1.5.3
-	github.com/adshao/go-binance/v2 v2.3.8
+	github.com/adshao/go-binance/v2 v2.3.10
 	github.com/c-bata/goptuna v0.8.1
 	github.com/c9s/requestgen v1.3.0
 	github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b

go.sum

@@ -53,6 +53,8 @@ github.com/adshao/go-binance/v2 v2.3.5 h1:WVYZecm0w8l14YoWlnKZj6xxZT2AKMTHpMQSqI
 github.com/adshao/go-binance/v2 v2.3.5/go.mod h1:8Pg/FGTLyAhq8QXA0IkoReKyRpoxJcK3LVujKDAZV/c=
 github.com/adshao/go-binance/v2 v2.3.8 h1:9VsAX4jUopnIOlzrvnKUFUf9SWB/nwPgJtUsM2dkj6A=
 github.com/adshao/go-binance/v2 v2.3.8/go.mod h1:Z3MCnWI0gHC4Rea8TWiF3aN1t4nV9z3CaU/TeHcKsLM=
+github.com/adshao/go-binance/v2 v2.3.10 h1:iWtHD/sQ8GK6r+cSMMdOynpGI/4Q6P5LZtiEHdWOjag=
+github.com/adshao/go-binance/v2 v2.3.10/go.mod h1:Z3MCnWI0gHC4Rea8TWiF3aN1t4nV9z3CaU/TeHcKsLM=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=

pkg/exchange/binance/binanceapi/get_my_trades_request.go

@@ -0,0 +1,27 @@
+package binanceapi
+
+import (
+	"time"
+
+	"github.com/c9s/requestgen"
+
+	"github.com/adshao/go-binance/v2"
+)
+
+type Trade = binance.TradeV3
+
+//go:generate requestgen -method GET -url "/api/v3/myTrades" -type GetMyTradesRequest -responseType []Trade
+type GetMyTradesRequest struct {
+	client requestgen.AuthenticatedAPIClient
+
+	symbol    string     `param:"symbol"`
+	orderID   *uint64    `param:"orderId"`
+	startTime *time.Time `param:"startTime,milliseconds"`
+	endTime   *time.Time `param:"endTime,milliseconds"`
+	fromID    *uint64    `param:"fromId"`
+	limit     *uint64    `param:"limit"`
+}
+
+func (c *RestClient) NewGetMyTradesRequest() *GetMyTradesRequest {
+	return &GetMyTradesRequest{client: c}
+}

pkg/exchange/binance/binanceapi/get_my_trades_request_requestgen.go

@@ -0,0 +1,218 @@
+// Code generated by "requestgen -method GET -url /api/v3/myTrades -type GetMyTradesRequest -responseType []Trade"; DO NOT EDIT.
+
+package binanceapi
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/adshao/go-binance/v2"
+	"net/url"
+	"reflect"
+	"regexp"
+	"strconv"
+	"time"
+)
+
+func (g *GetMyTradesRequest) Symbol(symbol string) *GetMyTradesRequest {
+	g.symbol = symbol
+	return g
+}
+
+func (g *GetMyTradesRequest) OrderID(orderID uint64) *GetMyTradesRequest {
+	g.orderID = &orderID
+	return g
+}
+
+func (g *GetMyTradesRequest) StartTime(startTime time.Time) *GetMyTradesRequest {
+	g.startTime = &startTime
+	return g
+}
+
+func (g *GetMyTradesRequest) EndTime(endTime time.Time) *GetMyTradesRequest {
+	g.endTime = &endTime
+	return g
+}
+
+func (g *GetMyTradesRequest) FromID(fromID uint64) *GetMyTradesRequest {
+	g.fromID = &fromID
+	return g
+}
+
+func (g *GetMyTradesRequest) Limit(limit uint64) *GetMyTradesRequest {
+	g.limit = &limit
+	return g
+}
+
+// GetQueryParameters builds and checks the query parameters and returns url.Values
+func (g *GetMyTradesRequest) GetQueryParameters() (url.Values, error) {
+	var params = map[string]interface{}{}
+
+	query := url.Values{}
+	for _k, _v := range params {
+		query.Add(_k, fmt.Sprintf("%v", _v))
+	}
+
+	return query, nil
+}
+
+// GetParameters builds and checks the parameters and return the result in a map object
+func (g *GetMyTradesRequest) GetParameters() (map[string]interface{}, error) {
+	var params = map[string]interface{}{}
+	// check symbol field -> json key symbol
+	symbol := g.symbol
+
+	// assign parameter of symbol
+	params["symbol"] = symbol
+	// check orderID field -> json key orderId
+	if g.orderID != nil {
+		orderID := *g.orderID
+
+		// assign parameter of orderID
+		params["orderId"] = orderID
+	} else {
+	}
+	// check startTime field -> json key startTime
+	if g.startTime != nil {
+		startTime := *g.startTime
+
+		// assign parameter of startTime
+		// convert time.Time to milliseconds time stamp
+		params["startTime"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10)
+	} else {
+	}
+	// check endTime field -> json key endTime
+	if g.endTime != nil {
+		endTime := *g.endTime
+
+		// assign parameter of endTime
+		// convert time.Time to milliseconds time stamp
+		params["endTime"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10)
+	} else {
+	}
+	// check fromID field -> json key fromId
+	if g.fromID != nil {
+		fromID := *g.fromID
+
+		// assign parameter of fromID
+		params["fromId"] = fromID
+	} else {
+	}
+	// check limit field -> json key limit
+	if g.limit != nil {
+		limit := *g.limit
+
+		// assign parameter of limit
+		params["limit"] = limit
+	} else {
+	}
+
+	return params, nil
+}
+
+// GetParametersQuery converts the parameters from GetParameters into the url.Values format
+func (g *GetMyTradesRequest) GetParametersQuery() (url.Values, error) {
+	query := url.Values{}
+
+	params, err := g.GetParameters()
+	if err != nil {
+		return query, err
+	}
+
+	for _k, _v := range params {
+		if g.isVarSlice(_v) {
+			g.iterateSlice(_v, func(it interface{}) {
+				query.Add(_k+"[]", fmt.Sprintf("%v", it))
+			})
+		} else {
+			query.Add(_k, fmt.Sprintf("%v", _v))
+		}
+	}
+
+	return query, nil
+}
+
+// GetParametersJSON converts the parameters from GetParameters into the JSON format
+func (g *GetMyTradesRequest) GetParametersJSON() ([]byte, error) {
+	params, err := g.GetParameters()
+	if err != nil {
+		return nil, err
+	}
+
+	return json.Marshal(params)
+}
+
+// GetSlugParameters builds and checks the slug parameters and return the result in a map object
+func (g *GetMyTradesRequest) GetSlugParameters() (map[string]interface{}, error) {
+	var params = map[string]interface{}{}
+
+	return params, nil
+}
+
+func (g *GetMyTradesRequest) applySlugsToUrl(url string, slugs map[string]string) string {
+	for _k, _v := range slugs {
+		needleRE := regexp.MustCompile(":" + _k + "\\b")
+		url = needleRE.ReplaceAllString(url, _v)
+	}
+
+	return url
+}
+
+func (g *GetMyTradesRequest) iterateSlice(slice interface{}, _f func(it interface{})) {
+	sliceValue := reflect.ValueOf(slice)
+	for _i := 0; _i < sliceValue.Len(); _i++ {
+		it := sliceValue.Index(_i).Interface()
+		_f(it)
+	}
+}
+
+func (g *GetMyTradesRequest) isVarSlice(_v interface{}) bool {
+	rt := reflect.TypeOf(_v)
+	switch rt.Kind() {
+	case reflect.Slice:
+		return true
+	}
+	return false
+}
+
+func (g *GetMyTradesRequest) GetSlugsMap() (map[string]string, error) {
+	slugs := map[string]string{}
+	params, err := g.GetSlugParameters()
+	if err != nil {
+		return slugs, nil
+	}
+
+	for _k, _v := range params {
+		slugs[_k] = fmt.Sprintf("%v", _v)
+	}
+
+	return slugs, nil
+}
+
+func (g *GetMyTradesRequest) Do(ctx context.Context) ([]binance.TradeV3, error) {
+
+	// empty params for GET operation
+	var params interface{}
+	query, err := g.GetParametersQuery()
+	if err != nil {
+		return nil, err
+	}
+
+	apiURL := "/api/v3/myTrades"
+
+	req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := g.client.SendRequest(req)
+	if err != nil {
+		return nil, err
+	}
+
+	var apiResponse []binance.TradeV3
+	if err := response.DecodeJSON(&apiResponse); err != nil {
+		return nil, err
+	}
+	return apiResponse, nil
+}

pkg/exchange/binance/exchange.go

@@ -1413,22 +1413,23 @@ func (e *Exchange) queryMarginTrades(ctx context.Context, symbol string, options
 		req.Limit(1000)
 	}
 
+	// BINANCE seems to have an API bug, we can't use both fromId and the start time/end time
 	// BINANCE uses inclusive last trade ID
 	if options.LastTradeID > 0 {
 		req.FromID(int64(options.LastTradeID))
-	}
-
-	if options.StartTime != nil && options.EndTime != nil {
-		if options.EndTime.Sub(*options.StartTime) < 24*time.Hour {
+	} else {
+		if options.StartTime != nil && options.EndTime != nil {
+			if options.EndTime.Sub(*options.StartTime) < 24*time.Hour {
+				req.StartTime(options.StartTime.UnixMilli())
+				req.EndTime(options.EndTime.UnixMilli())
+			} else {
+				req.StartTime(options.StartTime.UnixMilli())
+			}
+		} else if options.StartTime != nil {
 			req.StartTime(options.StartTime.UnixMilli())
+		} else if options.EndTime != nil {
 			req.EndTime(options.EndTime.UnixMilli())
-		} else {
-			req.StartTime(options.StartTime.UnixMilli())
 		}
-	} else if options.StartTime != nil {
-		req.StartTime(options.StartTime.UnixMilli())
-	} else if options.EndTime != nil {
-		req.EndTime(options.EndTime.UnixMilli())
 	}
 
 	remoteTrades, err = req.Do(ctx)
@@ -1499,40 +1500,40 @@ func (e *Exchange) queryFuturesTrades(ctx context.Context, symbol string, option
 }
 
 func (e *Exchange) querySpotTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) {
-	var remoteTrades []*binance.TradeV3
-	req := e.client.NewListTradesService().
-		Symbol(symbol)
+	req := e.client2.NewGetMyTradesRequest()
+	req.Symbol(symbol)
+
+	// BINANCE uses inclusive last trade ID
+	if options.LastTradeID > 0 {
+		req.FromID(options.LastTradeID)
+	} else {
+		if options.StartTime != nil && options.EndTime != nil {
+			if options.EndTime.Sub(*options.StartTime) < 24*time.Hour {
+				req.StartTime(*options.StartTime)
+				req.EndTime(*options.EndTime)
+			} else {
+				req.StartTime(*options.StartTime)
+			}
+		} else if options.StartTime != nil {
+			req.StartTime(*options.StartTime)
+		} else if options.EndTime != nil {
+			req.EndTime(*options.EndTime)
+		}
+	}
 
 	if options.Limit > 0 {
-		req.Limit(int(options.Limit))
+		req.Limit(uint64(options.Limit))
 	} else {
 		req.Limit(1000)
 	}
 
-	// BINANCE uses inclusive last trade ID
-	if options.LastTradeID > 0 {
-		req.FromID(int64(options.LastTradeID))
-	}
-
-	if options.StartTime != nil && options.EndTime != nil {
-		if options.EndTime.Sub(*options.StartTime) < 24*time.Hour {
-			req.StartTime(options.StartTime.UnixMilli())
-			req.EndTime(options.EndTime.UnixMilli())
-		} else {
-			req.StartTime(options.StartTime.UnixMilli())
-		}
-	} else if options.StartTime != nil {
-		req.StartTime(options.StartTime.UnixMilli())
-	} else if options.EndTime != nil {
-		req.EndTime(options.EndTime.UnixMilli())
-	}
-
-	remoteTrades, err = req.Do(ctx)
+	remoteTrades, err := req.Do(ctx)
 	if err != nil {
 		return nil, err
 	}
+
 	for _, t := range remoteTrades {
-		localTrade, err := toGlobalTrade(*t, e.IsMargin)
+		localTrade, err := toGlobalTrade(t, e.IsMargin)
 		if err != nil {
 			log.WithError(err).Errorf("can not convert binance trade: %+v", t)
 			continue

pkg/strategy/grid2/strategy_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/types"
 	"github.com/c9s/bbgo/pkg/types/mocks"
+	"github.com/c9s/bbgo/pkg/util"
 
 	gridmocks "github.com/c9s/bbgo/pkg/strategy/grid2/mocks"
 )
@@ -816,6 +817,11 @@ func TestStrategy_checkMinimalQuoteInvestment(t *testing.T) {
 }
 
 func TestBacktestStrategy(t *testing.T) {
+	if v, ok := util.GetEnvVarBool("TEST_BACKTEST"); !ok || !v {
+		t.Skip("backtest flag is required")
+		return
+	}
+
 	market := types.Market{
 		BaseCurrency:    "BTC",
 		QuoteCurrency:   "USDT",