Merge pull request #1255 from bailantaotao/edwin/query-trades

FEATURE: [bybit] add query trade api

pkg/exchange/bybit/bybitapi/v3/client.go

@@ -0,0 +1,13 @@
+package v3
+
+import (
+	"github.com/c9s/requestgen"
+
+	"github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi"
+)
+
+type APIResponse = bybitapi.APIResponse
+
+type Client struct {
+	Client requestgen.AuthenticatedAPIClient
+}

pkg/exchange/bybit/bybitapi/v3/client_test.go

@@ -0,0 +1,44 @@
+package v3
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi"
+	"github.com/c9s/bbgo/pkg/testutil"
+)
+
+func getTestClientOrSkip(t *testing.T) *bybitapi.RestClient {
+	if b, _ := strconv.ParseBool(os.Getenv("CI")); b {
+		t.Skip("skip test for CI")
+	}
+
+	key, secret, ok := testutil.IntegrationTestConfigured(t, "BYBIT")
+	if !ok {
+		t.Skip("BYBIT_* env vars are not configured")
+		return nil
+	}
+
+	client, err := bybitapi.NewClient()
+	assert.NoError(t, err)
+	client.Auth(key, secret)
+	return client
+}
+
+func TestClient(t *testing.T) {
+	client := getTestClientOrSkip(t)
+	v3Client := Client{Client: client}
+	ctx := context.Background()
+
+	t.Run("GetTradeRequest", func(t *testing.T) {
+		startTime := time.Date(2023, 7, 27, 16, 13, 9, 0, time.UTC)
+		apiResp, err := v3Client.NewGetTradesRequest().Symbol("BTCUSDT").StartTime(startTime).Do(ctx)
+		assert.NoError(t, err)
+		t.Logf("apiResp: %+v", apiResp)
+	})
+}

pkg/exchange/bybit/bybitapi/v3/get_trades_request.go

@@ -0,0 +1,55 @@
+package v3
+
+import (
+	"time"
+
+	"github.com/c9s/requestgen"
+
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Result
+//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Result
+
+type TradesResponse struct {
+	List []Trade `json:"list"`
+}
+
+type Trade struct {
+	Symbol        string                     `json:"symbol"`
+	Id            string                     `json:"id"`
+	OrderId       string                     `json:"orderId"`
+	TradeId       string                     `json:"tradeId"`
+	OrderPrice    fixedpoint.Value           `json:"orderPrice"`
+	OrderQty      fixedpoint.Value           `json:"orderQty"`
+	ExecFee       fixedpoint.Value           `json:"execFee"`
+	FeeTokenId    string                     `json:"feeTokenId"`
+	CreatTime     types.MillisecondTimestamp `json:"creatTime"`
+	IsBuyer       Side                       `json:"isBuyer"`
+	IsMaker       OrderType                  `json:"isMaker"`
+	MatchOrderId  string                     `json:"matchOrderId"`
+	MakerRebate   fixedpoint.Value           `json:"makerRebate"`
+	ExecutionTime types.MillisecondTimestamp `json:"executionTime"`
+	BlockTradeId  string                     `json:"blockTradeId"`
+}
+
+//go:generate GetRequest -url "/spot/v3/private/my-trades" -type GetTradesRequest -responseDataType .TradesResponse
+type GetTradesRequest struct {
+	client requestgen.AuthenticatedAPIClient
+
+	symbol  *string `param:"symbol,query"`
+	orderId *string `param:"orderId,query"`
+	// Limit default value is 50, max 50
+	limit       *uint64    `param:"limit,query"`
+	startTime   *time.Time `param:"startTime,query,milliseconds"`
+	endTime     *time.Time `param:"endTime,query,milliseconds"`
+	fromTradeId *string    `param:"fromTradeId,query"`
+	toTradeId   *string    `param:"toTradeId,query"`
+}
+
+func (c *Client) NewGetTradesRequest() *GetTradesRequest {
+	return &GetTradesRequest{
+		client: c.Client,
+	}
+}

pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go

@@ -0,0 +1,238 @@
+// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Result -url /spot/v3/private/my-trades -type GetTradesRequest -responseDataType .TradesResponse"; DO NOT EDIT.
+
+package v3
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi"
+	"net/url"
+	"reflect"
+	"regexp"
+	"strconv"
+	"time"
+)
+
+func (g *GetTradesRequest) Symbol(symbol string) *GetTradesRequest {
+	g.symbol = &symbol
+	return g
+}
+
+func (g *GetTradesRequest) OrderId(orderId string) *GetTradesRequest {
+	g.orderId = &orderId
+	return g
+}
+
+func (g *GetTradesRequest) Limit(limit uint64) *GetTradesRequest {
+	g.limit = &limit
+	return g
+}
+
+func (g *GetTradesRequest) StartTime(startTime time.Time) *GetTradesRequest {
+	g.startTime = &startTime
+	return g
+}
+
+func (g *GetTradesRequest) EndTime(endTime time.Time) *GetTradesRequest {
+	g.endTime = &endTime
+	return g
+}
+
+func (g *GetTradesRequest) FromTradeId(fromTradeId string) *GetTradesRequest {
+	g.fromTradeId = &fromTradeId
+	return g
+}
+
+func (g *GetTradesRequest) ToTradeId(toTradeId string) *GetTradesRequest {
+	g.toTradeId = &toTradeId
+	return g
+}
+
+// GetQueryParameters builds and checks the query parameters and returns url.Values
+func (g *GetTradesRequest) GetQueryParameters() (url.Values, error) {
+	var params = map[string]interface{}{}
+	// check symbol field -> json key symbol
+	if g.symbol != nil {
+		symbol := *g.symbol
+
+		// assign parameter of symbol
+		params["symbol"] = symbol
+	} else {
+	}
+	// check orderId field -> json key orderId
+	if g.orderId != nil {
+		orderId := *g.orderId
+
+		// assign parameter of orderId
+		params["orderId"] = orderId
+	} else {
+	}
+	// check limit field -> json key limit
+	if g.limit != nil {
+		limit := *g.limit
+
+		// assign parameter of limit
+		params["limit"] = limit
+	} else {
+	}
+	// check startTime field -> json key startTime
+	if g.startTime != nil {
+		startTime := *g.startTime
+
+		// assign parameter of startTime
+		// convert time.Time to milliseconds time stamp
+		params["startTime"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10)
+	} else {
+	}
+	// check endTime field -> json key endTime
+	if g.endTime != nil {
+		endTime := *g.endTime
+
+		// assign parameter of endTime
+		// convert time.Time to milliseconds time stamp
+		params["endTime"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10)
+	} else {
+	}
+	// check fromTradeId field -> json key fromTradeId
+	if g.fromTradeId != nil {
+		fromTradeId := *g.fromTradeId
+
+		// assign parameter of fromTradeId
+		params["fromTradeId"] = fromTradeId
+	} else {
+	}
+	// check toTradeId field -> json key toTradeId
+	if g.toTradeId != nil {
+		toTradeId := *g.toTradeId
+
+		// assign parameter of toTradeId
+		params["toTradeId"] = toTradeId
+	} else {
+	}
+
+	query := url.Values{}
+	for _k, _v := range params {
+		query.Add(_k, fmt.Sprintf("%v", _v))
+	}
+
+	return query, nil
+}
+
+// GetParameters builds and checks the parameters and return the result in a map object
+func (g *GetTradesRequest) GetParameters() (map[string]interface{}, error) {
+	var params = map[string]interface{}{}
+
+	return params, nil
+}
+
+// GetParametersQuery converts the parameters from GetParameters into the url.Values format
+func (g *GetTradesRequest) GetParametersQuery() (url.Values, error) {
+	query := url.Values{}
+
+	params, err := g.GetParameters()
+	if err != nil {
+		return query, err
+	}
+
+	for _k, _v := range params {
+		if g.isVarSlice(_v) {
+			g.iterateSlice(_v, func(it interface{}) {
+				query.Add(_k+"[]", fmt.Sprintf("%v", it))
+			})
+		} else {
+			query.Add(_k, fmt.Sprintf("%v", _v))
+		}
+	}
+
+	return query, nil
+}
+
+// GetParametersJSON converts the parameters from GetParameters into the JSON format
+func (g *GetTradesRequest) GetParametersJSON() ([]byte, error) {
+	params, err := g.GetParameters()
+	if err != nil {
+		return nil, err
+	}
+
+	return json.Marshal(params)
+}
+
+// GetSlugParameters builds and checks the slug parameters and return the result in a map object
+func (g *GetTradesRequest) GetSlugParameters() (map[string]interface{}, error) {
+	var params = map[string]interface{}{}
+
+	return params, nil
+}
+
+func (g *GetTradesRequest) applySlugsToUrl(url string, slugs map[string]string) string {
+	for _k, _v := range slugs {
+		needleRE := regexp.MustCompile(":" + _k + "\\b")
+		url = needleRE.ReplaceAllString(url, _v)
+	}
+
+	return url
+}
+
+func (g *GetTradesRequest) iterateSlice(slice interface{}, _f func(it interface{})) {
+	sliceValue := reflect.ValueOf(slice)
+	for _i := 0; _i < sliceValue.Len(); _i++ {
+		it := sliceValue.Index(_i).Interface()
+		_f(it)
+	}
+}
+
+func (g *GetTradesRequest) isVarSlice(_v interface{}) bool {
+	rt := reflect.TypeOf(_v)
+	switch rt.Kind() {
+	case reflect.Slice:
+		return true
+	}
+	return false
+}
+
+func (g *GetTradesRequest) GetSlugsMap() (map[string]string, error) {
+	slugs := map[string]string{}
+	params, err := g.GetSlugParameters()
+	if err != nil {
+		return slugs, nil
+	}
+
+	for _k, _v := range params {
+		slugs[_k] = fmt.Sprintf("%v", _v)
+	}
+
+	return slugs, nil
+}
+
+func (g *GetTradesRequest) Do(ctx context.Context) (*TradesResponse, error) {
+
+	// no body params
+	var params interface{}
+	query, err := g.GetQueryParameters()
+	if err != nil {
+		return nil, err
+	}
+
+	apiURL := "/spot/v3/private/my-trades"
+
+	req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := g.client.SendRequest(req)
+	if err != nil {
+		return nil, err
+	}
+
+	var apiResponse bybitapi.APIResponse
+	if err := response.DecodeJSON(&apiResponse); err != nil {
+		return nil, err
+	}
+	var data TradesResponse
+	if err := json.Unmarshal(apiResponse.Result, &data); err != nil {
+		return nil, err
+	}
+	return &data, nil
+}

pkg/exchange/bybit/bybitapi/v3/types.go

@@ -0,0 +1,15 @@
+package v3
+
+type Side string
+
+const (
+	SideBuy  Side = "0"
+	SideSell Side = "1"
+)
+
+type OrderType string
+
+const (
+	OrderTypeMaker OrderType = "0"
+	OrderTypeTaker OrderType = "1"
+)

pkg/exchange/bybit/convert.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi"
+	"github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi/v3"
 	"github.com/c9s/bbgo/pkg/types"
 )
 
@@ -73,7 +74,7 @@ func toGlobalOrder(order bybitapi.Order) (*types.Order, error) {
 	// Now we only use spot trading.
 	orderIdNum, err := strconv.ParseUint(order.OrderId, 10, 64)
 	if err != nil {
-		return nil, fmt.Errorf("unexpected order id: %s, err: %v", order.OrderId, err)
+		return nil, fmt.Errorf("unexpected order id: %s, err: %w", order.OrderId, err)
 	}
 
 	return &types.Order{
@@ -204,3 +205,63 @@ func toLocalSide(side types.SideType) (bybitapi.Side, error) {
 		return "", fmt.Errorf("side type %s not supported", side)
 	}
 }
+
+func toV3Buyer(isBuyer v3.Side) (types.SideType, error) {
+	switch isBuyer {
+	case v3.SideBuy:
+		return types.SideTypeBuy, nil
+	case v3.SideSell:
+		return types.SideTypeSell, nil
+	default:
+		return "", fmt.Errorf("unexpected side type: %s", isBuyer)
+	}
+}
+func toV3Maker(isMaker v3.OrderType) (bool, error) {
+	switch isMaker {
+	case v3.OrderTypeMaker:
+		return true, nil
+	case v3.OrderTypeTaker:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected order type: %s", isMaker)
+	}
+}
+
+func v3ToGlobalTrade(trade v3.Trade) (*types.Trade, error) {
+	side, err := toV3Buyer(trade.IsBuyer)
+	if err != nil {
+		return nil, err
+	}
+	isMaker, err := toV3Maker(trade.IsMaker)
+	if err != nil {
+		return nil, err
+	}
+
+	orderIdNum, err := strconv.ParseUint(trade.OrderId, 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected order id: %s, err: %w", trade.OrderId, err)
+	}
+	tradeIdNum, err := strconv.ParseUint(trade.TradeId, 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected trade id: %s, err: %w", trade.TradeId, err)
+	}
+
+	return &types.Trade{
+		ID:            tradeIdNum,
+		OrderID:       orderIdNum,
+		Exchange:      types.ExchangeBybit,
+		Price:         trade.OrderPrice,
+		Quantity:      trade.OrderQty,
+		QuoteQuantity: trade.OrderPrice.Mul(trade.OrderQty),
+		Symbol:        trade.Symbol,
+		Side:          side,
+		IsBuyer:       side == types.SideTypeBuy,
+		IsMaker:       isMaker,
+		Time:          types.Time(trade.ExecutionTime),
+		Fee:           trade.ExecFee,
+		FeeCurrency:   trade.FeeTokenId,
+		IsMargin:      false,
+		IsFutures:     false,
+		IsIsolated:    false,
+	}, nil
+}

pkg/exchange/bybit/convert_test.go

@@ -3,16 +3,17 @@ package bybit
 import (
 	"context"
 	"fmt"
-	"github.com/pkg/errors"
-	"go.uber.org/multierr"
 	"math"
 	"strconv"
 	"testing"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
+	"go.uber.org/multierr"
 
 	"github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi"
+	v3 "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi/v3"
 	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/types"
 )
@@ -403,3 +404,74 @@ func Test_toLocalSide(t *testing.T) {
 	assert.Error(t, fmt.Errorf("side type %s not supported", "wrong side"), err)
 	assert.Equal(t, bybitapi.Side(""), side)
 }
+
+func Test_toGlobalTrade(t *testing.T) {
+	/* sample: trade
+	{
+		"Symbol":"BTCUSDT",
+		"Id":"1474200510090276864",
+		"OrderId":"1474200270671015936",
+		"TradeId":"2100000000031181772",
+		"OrderPrice":"27628",
+		"OrderQty":"0.007959",
+		"ExecFee":"0.21989125",
+		"FeeTokenId":"USDT",
+		"CreatTime":"2023-07-28 00:13:15.457 +0800 CST",
+		"IsBuyer":"1",
+		"IsMaker":"0",
+		"MatchOrderId":"5760912963729109504",
+		"MakerRebate":"0",
+		"ExecutionTime":"2023-07-28 00:13:15.463 +0800 CST",
+		"BlockTradeId": "",
+	}
+	*/
+	timeNow := time.Now()
+	trade := v3.Trade{
+		Symbol:        "DOTUSDT",
+		Id:            "1474200510090276864",
+		OrderId:       "1474200270671015936",
+		TradeId:       "2100000000031181772",
+		OrderPrice:    fixedpoint.NewFromFloat(27628),
+		OrderQty:      fixedpoint.NewFromFloat(0.007959),
+		ExecFee:       fixedpoint.NewFromFloat(0.21989125),
+		FeeTokenId:    "USDT",
+		CreatTime:     types.MillisecondTimestamp(timeNow),
+		IsBuyer:       "0",
+		IsMaker:       "0",
+		MatchOrderId:  "5760912963729109504",
+		MakerRebate:   fixedpoint.NewFromFloat(0),
+		ExecutionTime: types.MillisecondTimestamp(timeNow),
+		BlockTradeId:  "",
+	}
+
+	s, err := toV3Buyer(trade.IsBuyer)
+	assert.NoError(t, err)
+	m, err := toV3Maker(trade.IsMaker)
+	assert.NoError(t, err)
+	orderIdNum, err := strconv.ParseUint(trade.OrderId, 10, 64)
+	assert.NoError(t, err)
+	tradeId, err := strconv.ParseUint(trade.TradeId, 10, 64)
+	assert.NoError(t, err)
+
+	exp := types.Trade{
+		ID:            tradeId,
+		OrderID:       orderIdNum,
+		Exchange:      types.ExchangeBybit,
+		Price:         trade.OrderPrice,
+		Quantity:      trade.OrderQty,
+		QuoteQuantity: trade.OrderPrice.Mul(trade.OrderQty),
+		Symbol:        trade.Symbol,
+		Side:          s,
+		IsBuyer:       s == types.SideTypeBuy,
+		IsMaker:       m,
+		Time:          types.Time(timeNow),
+		Fee:           trade.ExecFee,
+		FeeCurrency:   trade.FeeTokenId,
+		IsMargin:      false,
+		IsFutures:     false,
+		IsIsolated:    false,
+	}
+	res, err := v3ToGlobalTrade(trade)
+	assert.NoError(t, err)
+	assert.Equal(t, res, &exp)
+}

pkg/exchange/bybit/exchange.go

@@ -11,12 +11,15 @@ import (
 	"golang.org/x/time/rate"
 
 	"github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi"
+	v3 "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi/v3"
 	"github.com/c9s/bbgo/pkg/types"
 )
 
 const (
-	maxOrderIdLen         = 36
-	defaultQueryClosedLen = 50
+	maxOrderIdLen     = 36
+	defaultQueryLimit = 50
+
+	halfYearDuration = 6 * 30 * 24 * time.Hour
 )
 
 // https://bybit-exchange.github.io/docs/zh-TW/v5/rate-limit
@@ -25,10 +28,10 @@ const (
 // The default order limiter apply 2 requests per second and a 2 initial bucket
 // this includes QueryMarkets, QueryTicker
 var (
-	sharedRateLimiter = rate.NewLimiter(rate.Every(time.Second/2), 2)
-	tradeRateLimiter  = rate.NewLimiter(rate.Every(time.Second/5), 5)
-	orderRateLimiter  = rate.NewLimiter(rate.Every(100*time.Millisecond), 10)
-	closedRateLimiter = rate.NewLimiter(rate.Every(time.Second), 1)
+	sharedRateLimiter       = rate.NewLimiter(rate.Every(time.Second/2), 2)
+	tradeRateLimiter        = rate.NewLimiter(rate.Every(time.Second/5), 5)
+	orderRateLimiter        = rate.NewLimiter(rate.Every(100*time.Millisecond), 10)
+	closedOrderQueryLimiter = rate.NewLimiter(rate.Every(time.Second), 1)
 
 	log = logrus.WithFields(logrus.Fields{
 		"exchange": "bybit",
@@ -38,6 +41,7 @@ var (
 type Exchange struct {
 	key, secret string
 	client      *bybitapi.RestClient
+	v3client    *v3.Client
 }
 
 func New(key, secret string) (*Exchange, error) {
@@ -286,13 +290,13 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since,
 		log.Warn("!!!BYBIT EXCHANGE API NOTICE!!! the since/until conditions will not be effected on SPOT account, bybit exchange does not support time-range-based query currently")
 	}
 
-	if err := closedRateLimiter.Wait(ctx); err != nil {
+	if err := closedOrderQueryLimiter.Wait(ctx); err != nil {
 		return nil, fmt.Errorf("query closed order rate limiter wait error: %w", err)
 	}
 	res, err := e.client.NewGetOrderHistoriesRequest().
 		Symbol(symbol).
 		Cursor(strconv.FormatUint(lastOrderID, 10)).
-		Limit(defaultQueryClosedLen).
+		Limit(defaultQueryLimit).
 		Do(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to call get order histories error: %w", err)
@@ -315,3 +319,71 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since,
 
 	return types.SortOrdersAscending(orders), nil
 }
+
+/*
+QueryTrades queries trades by time range or trade id range.
+If options.StartTime is not specified, you can only query for records in the last 7 days.
+If you want to query for records older than 7 days, options.StartTime is required.
+It supports to query records up to 180 days.
+
+If the orderId is null, fromTradeId is passed, and toTradeId is null, then the result is sorted by
+ticketId in ascend. Otherwise, the result is sorted by ticketId in descend.
+
+** Here includes MakerRebate. If needed, let's discuss how to modify it to return in trade. **
+** StartTime and EndTime are inclusive. **
+** StartTime and EndTime cannot exceed 180 days. **
+*/
+func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) {
+	if options.StartTime != nil && options.EndTime != nil && options.EndTime.Sub(*options.StartTime) > halfYearDuration {
+		return nil, fmt.Errorf("StartTime and EndTime cannot exceed 180 days, startTime: %v, endTime: %v, diff: %v",
+			options.StartTime.String(),
+			options.EndTime.String(),
+			options.EndTime.Sub(*options.StartTime)/24)
+	}
+
+	// using v3 client, since the v5 API does not support feeCurrency.
+	req := e.v3client.NewGetTradesRequest()
+	req.Symbol(symbol)
+
+	if options.StartTime != nil || options.EndTime != nil {
+		if options.StartTime != nil {
+			req.StartTime(options.StartTime.UTC())
+		}
+		if options.EndTime != nil {
+			req.EndTime(options.EndTime.UTC())
+		}
+	} else {
+		req.FromTradeId(strconv.FormatUint(options.LastTradeID, 10))
+	}
+
+	limit := uint64(options.Limit)
+	if limit > defaultQueryLimit || limit <= 0 {
+		log.Debugf("limtit is exceeded or zero, update to %d, got: %d", defaultQueryLimit, options.Limit)
+		limit = defaultQueryLimit
+	}
+	req.Limit(limit)
+
+	if err := tradeRateLimiter.Wait(ctx); err != nil {
+		return nil, fmt.Errorf("trade rate limiter wait error: %w", err)
+	}
+	response, err := req.Do(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query trades, err: %w", err)
+	}
+
+	var errs error
+	for _, trade := range response.List {
+		res, err := v3ToGlobalTrade(trade)
+		if err != nil {
+			errs = multierr.Append(errs, err)
+			continue
+		}
+		trades = append(trades, *res)
+	}
+
+	if errs != nil {
+		return nil, errs
+	}
+
+	return trades, nil
+}