From 4363f0ae7bd70b62e739abbdc0232813b5addef9 Mon Sep 17 00:00:00 2001 From: Edwin Date: Fri, 28 Jul 2023 15:37:05 +0800 Subject: [PATCH] pkg/exchange: add query trade api --- pkg/exchange/bybit/bybitapi/v3/client.go | 13 + pkg/exchange/bybit/bybitapi/v3/client_test.go | 44 ++++ .../bybit/bybitapi/v3/get_trades_request.go | 55 ++++ .../v3/get_trades_request_requestgen.go | 238 ++++++++++++++++++ pkg/exchange/bybit/bybitapi/v3/types.go | 15 ++ pkg/exchange/bybit/convert.go | 63 ++++- pkg/exchange/bybit/convert_test.go | 76 +++++- pkg/exchange/bybit/exchange.go | 88 ++++++- 8 files changed, 581 insertions(+), 11 deletions(-) create mode 100644 pkg/exchange/bybit/bybitapi/v3/client.go create mode 100644 pkg/exchange/bybit/bybitapi/v3/client_test.go create mode 100644 pkg/exchange/bybit/bybitapi/v3/get_trades_request.go create mode 100644 pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go create mode 100644 pkg/exchange/bybit/bybitapi/v3/types.go diff --git a/pkg/exchange/bybit/bybitapi/v3/client.go b/pkg/exchange/bybit/bybitapi/v3/client.go new file mode 100644 index 000000000..9fdb889fb --- /dev/null +++ b/pkg/exchange/bybit/bybitapi/v3/client.go @@ -0,0 +1,13 @@ +package v3 + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" +) + +type APIResponse = bybitapi.APIResponse + +type Client struct { + Client requestgen.AuthenticatedAPIClient +} diff --git a/pkg/exchange/bybit/bybitapi/v3/client_test.go b/pkg/exchange/bybit/bybitapi/v3/client_test.go new file mode 100644 index 000000000..5bc075c62 --- /dev/null +++ b/pkg/exchange/bybit/bybitapi/v3/client_test.go @@ -0,0 +1,44 @@ +package v3 + +import ( + "context" + "os" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + "github.com/c9s/bbgo/pkg/testutil" +) + +func getTestClientOrSkip(t *testing.T) *bybitapi.RestClient { + if b, _ := strconv.ParseBool(os.Getenv("CI")); b { + t.Skip("skip test for CI") + } + + key, secret, ok := testutil.IntegrationTestConfigured(t, "BYBIT") + if !ok { + t.Skip("BYBIT_* env vars are not configured") + return nil + } + + client, err := bybitapi.NewClient() + assert.NoError(t, err) + client.Auth(key, secret) + return client +} + +func TestClient(t *testing.T) { + client := getTestClientOrSkip(t) + v3Client := Client{Client: client} + ctx := context.Background() + + t.Run("GetTradeRequest", func(t *testing.T) { + startTime := time.Date(2023, 7, 27, 16, 13, 9, 0, time.UTC) + apiResp, err := v3Client.NewGetTradesRequest().Symbol("BTCUSDT").StartTime(startTime).Do(ctx) + assert.NoError(t, err) + t.Logf("apiResp: %+v", apiResp) + }) +} diff --git a/pkg/exchange/bybit/bybitapi/v3/get_trades_request.go b/pkg/exchange/bybit/bybitapi/v3/get_trades_request.go new file mode 100644 index 000000000..7f91487d1 --- /dev/null +++ b/pkg/exchange/bybit/bybitapi/v3/get_trades_request.go @@ -0,0 +1,55 @@ +package v3 + +import ( + "time" + + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Result +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Result + +type TradesResponse struct { + List []Trade `json:"list"` +} + +type Trade struct { + Symbol string `json:"symbol"` + Id string `json:"id"` + OrderId string `json:"orderId"` + TradeId string `json:"tradeId"` + OrderPrice fixedpoint.Value `json:"orderPrice"` + OrderQty fixedpoint.Value `json:"orderQty"` + ExecFee fixedpoint.Value `json:"execFee"` + FeeTokenId string `json:"feeTokenId"` + CreatTime types.MillisecondTimestamp `json:"creatTime"` + IsBuyer Side `json:"isBuyer"` + IsMaker OrderType `json:"isMaker"` + MatchOrderId string `json:"matchOrderId"` + MakerRebate fixedpoint.Value `json:"makerRebate"` + ExecutionTime types.MillisecondTimestamp `json:"executionTime"` + BlockTradeId string `json:"blockTradeId"` +} + +//go:generate GetRequest -url "/spot/v3/private/my-trades" -type GetTradesRequest -responseDataType .TradesResponse +type GetTradesRequest struct { + client requestgen.AuthenticatedAPIClient + + symbol *string `param:"symbol,query"` + orderId *string `param:"orderId,query"` + // Limit default value is 50, max 50 + limit *uint64 `param:"limit,query"` + startTime *time.Time `param:"startTime,query,milliseconds"` + endTime *time.Time `param:"endTime,query,milliseconds"` + fromTradeId *string `param:"fromTradeId,query"` + toTradeId *string `param:"toTradeId,query"` +} + +func (c *Client) NewGetTradesRequest() *GetTradesRequest { + return &GetTradesRequest{ + client: c.Client, + } +} diff --git a/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go b/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go new file mode 100644 index 000000000..00d7dd80a --- /dev/null +++ b/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go @@ -0,0 +1,238 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Result -url /spot/v3/private/my-trades -type GetTradesRequest -responseDataType .TradesResponse"; DO NOT EDIT. + +package v3 + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + "net/url" + "reflect" + "regexp" + "strconv" + "time" +) + +func (g *GetTradesRequest) Symbol(symbol string) *GetTradesRequest { + g.symbol = &symbol + return g +} + +func (g *GetTradesRequest) OrderId(orderId string) *GetTradesRequest { + g.orderId = &orderId + return g +} + +func (g *GetTradesRequest) Limit(limit uint64) *GetTradesRequest { + g.limit = &limit + return g +} + +func (g *GetTradesRequest) StartTime(startTime time.Time) *GetTradesRequest { + g.startTime = &startTime + return g +} + +func (g *GetTradesRequest) EndTime(endTime time.Time) *GetTradesRequest { + g.endTime = &endTime + return g +} + +func (g *GetTradesRequest) FromTradeId(fromTradeId string) *GetTradesRequest { + g.fromTradeId = &fromTradeId + return g +} + +func (g *GetTradesRequest) ToTradeId(toTradeId string) *GetTradesRequest { + g.toTradeId = &toTradeId + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetTradesRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + if g.symbol != nil { + symbol := *g.symbol + + // assign parameter of symbol + params["symbol"] = symbol + } else { + } + // check orderId field -> json key orderId + if g.orderId != nil { + orderId := *g.orderId + + // assign parameter of orderId + params["orderId"] = orderId + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + // check startTime field -> json key startTime + if g.startTime != nil { + startTime := *g.startTime + + // assign parameter of startTime + // convert time.Time to milliseconds time stamp + params["startTime"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check endTime field -> json key endTime + if g.endTime != nil { + endTime := *g.endTime + + // assign parameter of endTime + // convert time.Time to milliseconds time stamp + params["endTime"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check fromTradeId field -> json key fromTradeId + if g.fromTradeId != nil { + fromTradeId := *g.fromTradeId + + // assign parameter of fromTradeId + params["fromTradeId"] = fromTradeId + } else { + } + // check toTradeId field -> json key toTradeId + if g.toTradeId != nil { + toTradeId := *g.toTradeId + + // assign parameter of toTradeId + params["toTradeId"] = toTradeId + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetTradesRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetTradesRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetTradesRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetTradesRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetTradesRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetTradesRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetTradesRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetTradesRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (g *GetTradesRequest) Do(ctx context.Context) (*TradesResponse, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + apiURL := "/spot/v3/private/my-trades" + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bybitapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + var data TradesResponse + if err := json.Unmarshal(apiResponse.Result, &data); err != nil { + return nil, err + } + return &data, nil +} diff --git a/pkg/exchange/bybit/bybitapi/v3/types.go b/pkg/exchange/bybit/bybitapi/v3/types.go new file mode 100644 index 000000000..c3f7a1b83 --- /dev/null +++ b/pkg/exchange/bybit/bybitapi/v3/types.go @@ -0,0 +1,15 @@ +package v3 + +type Side string + +const ( + SideBuy Side = "0" + SideSell Side = "1" +) + +type OrderType string + +const ( + OrderTypeMaker OrderType = "0" + OrderTypeTaker OrderType = "1" +) diff --git a/pkg/exchange/bybit/convert.go b/pkg/exchange/bybit/convert.go index b70a05201..c78f6dc1b 100644 --- a/pkg/exchange/bybit/convert.go +++ b/pkg/exchange/bybit/convert.go @@ -8,6 +8,7 @@ import ( "time" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi/v3" "github.com/c9s/bbgo/pkg/types" ) @@ -73,7 +74,7 @@ func toGlobalOrder(order bybitapi.Order) (*types.Order, error) { // Now we only use spot trading. orderIdNum, err := strconv.ParseUint(order.OrderId, 10, 64) if err != nil { - return nil, fmt.Errorf("unexpected order id: %s, err: %v", order.OrderId, err) + return nil, fmt.Errorf("unexpected order id: %s, err: %w", order.OrderId, err) } return &types.Order{ @@ -204,3 +205,63 @@ func toLocalSide(side types.SideType) (bybitapi.Side, error) { return "", fmt.Errorf("side type %s not supported", side) } } + +func toV3Buyer(isBuyer v3.Side) (types.SideType, error) { + switch isBuyer { + case v3.SideBuy: + return types.SideTypeBuy, nil + case v3.SideSell: + return types.SideTypeSell, nil + default: + return "", fmt.Errorf("unexpected side type: %s", isBuyer) + } +} +func toV3Maker(isMaker v3.OrderType) (bool, error) { + switch isMaker { + case v3.OrderTypeMaker: + return true, nil + case v3.OrderTypeTaker: + return false, nil + default: + return false, fmt.Errorf("unexpected order type: %s", isMaker) + } +} + +func v3ToGlobalTrade(trade v3.Trade) (*types.Trade, error) { + side, err := toV3Buyer(trade.IsBuyer) + if err != nil { + return nil, err + } + isMaker, err := toV3Maker(trade.IsMaker) + if err != nil { + return nil, err + } + + orderIdNum, err := strconv.ParseUint(trade.OrderId, 10, 64) + if err != nil { + return nil, fmt.Errorf("unexpected order id: %s, err: %w", trade.OrderId, err) + } + tradeIdNum, err := strconv.ParseUint(trade.TradeId, 10, 64) + if err != nil { + return nil, fmt.Errorf("unexpected trade id: %s, err: %w", trade.TradeId, err) + } + + return &types.Trade{ + ID: tradeIdNum, + OrderID: orderIdNum, + Exchange: types.ExchangeBybit, + Price: trade.OrderPrice, + Quantity: trade.OrderQty, + QuoteQuantity: trade.OrderPrice.Mul(trade.OrderQty), + Symbol: trade.Symbol, + Side: side, + IsBuyer: side == types.SideTypeBuy, + IsMaker: isMaker, + Time: types.Time(trade.ExecutionTime), + Fee: trade.ExecFee, + FeeCurrency: trade.FeeTokenId, + IsMargin: false, + IsFutures: false, + IsIsolated: false, + }, nil +} diff --git a/pkg/exchange/bybit/convert_test.go b/pkg/exchange/bybit/convert_test.go index e4bffcf30..3051a3376 100644 --- a/pkg/exchange/bybit/convert_test.go +++ b/pkg/exchange/bybit/convert_test.go @@ -3,16 +3,17 @@ package bybit import ( "context" "fmt" - "github.com/pkg/errors" - "go.uber.org/multierr" "math" "strconv" "testing" "time" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "go.uber.org/multierr" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + v3 "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi/v3" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -403,3 +404,74 @@ func Test_toLocalSide(t *testing.T) { assert.Error(t, fmt.Errorf("side type %s not supported", "wrong side"), err) assert.Equal(t, bybitapi.Side(""), side) } + +func Test_toGlobalTrade(t *testing.T) { + /* sample: trade + { + "Symbol":"BTCUSDT", + "Id":"1474200510090276864", + "OrderId":"1474200270671015936", + "TradeId":"2100000000031181772", + "OrderPrice":"27628", + "OrderQty":"0.007959", + "ExecFee":"0.21989125", + "FeeTokenId":"USDT", + "CreatTime":"2023-07-28 00:13:15.457 +0800 CST", + "IsBuyer":"1", + "IsMaker":"0", + "MatchOrderId":"5760912963729109504", + "MakerRebate":"0", + "ExecutionTime":"2023-07-28 00:13:15.463 +0800 CST", + "BlockTradeId": "", + } + */ + timeNow := time.Now() + trade := v3.Trade{ + Symbol: "DOTUSDT", + Id: "1474200510090276864", + OrderId: "1474200270671015936", + TradeId: "2100000000031181772", + OrderPrice: fixedpoint.NewFromFloat(27628), + OrderQty: fixedpoint.NewFromFloat(0.007959), + ExecFee: fixedpoint.NewFromFloat(0.21989125), + FeeTokenId: "USDT", + CreatTime: types.MillisecondTimestamp(timeNow), + IsBuyer: "0", + IsMaker: "0", + MatchOrderId: "5760912963729109504", + MakerRebate: fixedpoint.NewFromFloat(0), + ExecutionTime: types.MillisecondTimestamp(timeNow), + BlockTradeId: "", + } + + s, err := toV3Buyer(trade.IsBuyer) + assert.NoError(t, err) + m, err := toV3Maker(trade.IsMaker) + assert.NoError(t, err) + orderIdNum, err := strconv.ParseUint(trade.OrderId, 10, 64) + assert.NoError(t, err) + tradeId, err := strconv.ParseUint(trade.TradeId, 10, 64) + assert.NoError(t, err) + + exp := types.Trade{ + ID: tradeId, + OrderID: orderIdNum, + Exchange: types.ExchangeBybit, + Price: trade.OrderPrice, + Quantity: trade.OrderQty, + QuoteQuantity: trade.OrderPrice.Mul(trade.OrderQty), + Symbol: trade.Symbol, + Side: s, + IsBuyer: s == types.SideTypeBuy, + IsMaker: m, + Time: types.Time(timeNow), + Fee: trade.ExecFee, + FeeCurrency: trade.FeeTokenId, + IsMargin: false, + IsFutures: false, + IsIsolated: false, + } + res, err := v3ToGlobalTrade(trade) + assert.NoError(t, err) + assert.Equal(t, res, &exp) +} diff --git a/pkg/exchange/bybit/exchange.go b/pkg/exchange/bybit/exchange.go index 33a68f2dc..d838283f5 100644 --- a/pkg/exchange/bybit/exchange.go +++ b/pkg/exchange/bybit/exchange.go @@ -11,12 +11,15 @@ import ( "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + v3 "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi/v3" "github.com/c9s/bbgo/pkg/types" ) const ( - maxOrderIdLen = 36 - defaultQueryClosedLen = 50 + maxOrderIdLen = 36 + defaultQueryLimit = 50 + + halfYearDuration = 6 * 30 * 24 * time.Hour ) // https://bybit-exchange.github.io/docs/zh-TW/v5/rate-limit @@ -25,10 +28,10 @@ const ( // The default order limiter apply 2 requests per second and a 2 initial bucket // this includes QueryMarkets, QueryTicker var ( - sharedRateLimiter = rate.NewLimiter(rate.Every(time.Second/2), 2) - tradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) - orderRateLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) - closedRateLimiter = rate.NewLimiter(rate.Every(time.Second), 1) + sharedRateLimiter = rate.NewLimiter(rate.Every(time.Second/2), 2) + tradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + orderRateLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) + closedOrderQueryLimiter = rate.NewLimiter(rate.Every(time.Second), 1) log = logrus.WithFields(logrus.Fields{ "exchange": "bybit", @@ -38,6 +41,7 @@ var ( type Exchange struct { key, secret string client *bybitapi.RestClient + v3client *v3.Client } func New(key, secret string) (*Exchange, error) { @@ -286,13 +290,13 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, log.Warn("!!!BYBIT EXCHANGE API NOTICE!!! the since/until conditions will not be effected on SPOT account, bybit exchange does not support time-range-based query currently") } - if err := closedRateLimiter.Wait(ctx); err != nil { + if err := closedOrderQueryLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("query closed order rate limiter wait error: %w", err) } res, err := e.client.NewGetOrderHistoriesRequest(). Symbol(symbol). Cursor(strconv.FormatUint(lastOrderID, 10)). - Limit(defaultQueryClosedLen). + Limit(defaultQueryLimit). Do(ctx) if err != nil { return nil, fmt.Errorf("failed to call get order histories error: %w", err) @@ -315,3 +319,71 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, return types.SortOrdersAscending(orders), nil } + +/* +QueryTrades queries trades by time range or trade id range. +If options.StartTime is not specified, you can only query for records in the last 7 days. +If you want to query for records older than 7 days, options.StartTime is required. +It supports to query records up to 180 days. + +If the orderId is null, fromTradeId is passed, and toTradeId is null, then the result is sorted by +ticketId in ascend. Otherwise, the result is sorted by ticketId in descend. + +** Here includes MakerRebate. If needed, let's discuss how to modify it to return in trade. ** +** StartTime and EndTime are inclusive. ** +** StartTime and EndTime cannot exceed 180 days. ** +*/ +func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) { + if options.StartTime != nil && options.EndTime != nil && options.EndTime.Sub(*options.StartTime) > halfYearDuration { + return nil, fmt.Errorf("StartTime and EndTime cannot exceed 180 days, startTime: %v, endTime: %v, diff: %v", + options.StartTime.String(), + options.EndTime.String(), + options.EndTime.Sub(*options.StartTime)/24) + } + + // using v3 client, since the v5 API does not support feeCurrency. + req := e.v3client.NewGetTradesRequest() + req.Symbol(symbol) + + if options.StartTime != nil || options.EndTime != nil { + if options.StartTime != nil { + req.StartTime(options.StartTime.UTC()) + } + if options.EndTime != nil { + req.EndTime(options.EndTime.UTC()) + } + } else { + req.FromTradeId(strconv.FormatUint(options.LastTradeID, 10)) + } + + limit := uint64(options.Limit) + if limit > defaultQueryLimit || limit <= 0 { + log.Debugf("limtit is exceeded or zero, update to %d, got: %d", defaultQueryLimit, options.Limit) + limit = defaultQueryLimit + } + req.Limit(limit) + + if err := tradeRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("trade rate limiter wait error: %w", err) + } + response, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query trades, err: %w", err) + } + + var errs error + for _, trade := range response.List { + res, err := v3ToGlobalTrade(trade) + if err != nil { + errs = multierr.Append(errs, err) + continue + } + trades = append(trades, *res) + } + + if errs != nil { + return nil, errs + } + + return trades, nil +}