From 755ea5e427be734b503de6691e0a94d701820a2e Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 13 Nov 2023 19:21:58 +0800 Subject: [PATCH 1/2] pkg/exchange: implement query kline api --- .../bitget/bitgetapi/v2/client_test.go | 9 + .../get_history_orders_request_requestgen.go | 19 +- .../bitget/bitgetapi/v2/get_k_line.go | 83 +++++++ .../v2/get_k_line_request_requestgen.go | 224 ++++++++++++++++++ pkg/exchange/bitget/convert.go | 24 ++ pkg/exchange/bitget/convert_test.go | 86 +++++++ pkg/exchange/bitget/exchange.go | 60 ++++- pkg/exchange/bitget/types.go | 35 +++ pkg/exchange/bitget/types_test.go | 57 +++-- 9 files changed, 561 insertions(+), 36 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_k_line.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_k_line_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 97b836f7f..31909c7b7 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -5,6 +5,7 @@ import ( "os" "strconv" "testing" + "time" "github.com/stretchr/testify/assert" @@ -78,4 +79,12 @@ func TestClient(t *testing.T) { resp, err := client.NewCancelOrderRequest().Symbol("APEUSDT").OrderId(req.OrderId).Do(ctx) t.Logf("cancel order resp: %+v", resp) }) + + t.Run("GetKLineRequest", func(t *testing.T) { + startTime := time.Date(2023, 8, 12, 0, 0, 0, 0, time.UTC) + endTime := time.Date(2023, 10, 14, 0, 0, 0, 0, time.UTC) + resp, err := client.NewGetKLineRequest().Symbol("APEUSDT").Granularity("30min").StartTime(startTime).EndTime(endTime).Limit("1000").Do(ctx) + assert.NoError(t, err) + t.Logf("resp: %+v", resp) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go index 0a681f322..398093399 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go @@ -188,6 +188,12 @@ func (g *GetHistoryOrdersRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetHistoryOrdersRequest) GetPath() string { + return "/api/v2/spot/trade/history-orders" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetHistoryOrdersRequest) Do(ctx context.Context) ([]OrderDetail, error) { // no body params @@ -197,7 +203,9 @@ func (g *GetHistoryOrdersRequest) Do(ctx context.Context) ([]OrderDetail, error) return nil, err } - apiURL := "/api/v2/spot/trade/history-orders" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -214,6 +222,15 @@ func (g *GetHistoryOrdersRequest) Do(ctx context.Context) ([]OrderDetail, error) return nil, err } + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data []OrderDetail if err := json.Unmarshal(apiResponse.Data, &data); err != nil { return nil, err diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_k_line.go b/pkg/exchange/bitget/bitgetapi/v2/get_k_line.go new file mode 100644 index 000000000..3a80a7c2f --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_k_line.go @@ -0,0 +1,83 @@ +package bitgetapi + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type KLine struct { + // System timestamp, Unix millisecond timestamp, e.g. 1690196141868 + Ts types.MillisecondTimestamp + Open fixedpoint.Value + High fixedpoint.Value + Low fixedpoint.Value + Close fixedpoint.Value + // Trading volume in base currency, e.g. "BTC" in the "BTCUSD" pair. + Volume fixedpoint.Value + // Trading volume in quote currency, e.g. "USD" in the "BTCUSD" pair. + QuoteVolume fixedpoint.Value + // Trading volume in USDT + UsdtVolume fixedpoint.Value +} + +type KLineResponse []KLine + +const KLinesArrayLen = 8 + +func (k *KLine) UnmarshalJSON(data []byte) error { + var jsonArr []json.RawMessage + err := json.Unmarshal(data, &jsonArr) + if err != nil { + return fmt.Errorf("failed to unmarshal jsonRawMessage: %v, err: %w", string(data), err) + } + if len(jsonArr) != KLinesArrayLen { + return fmt.Errorf("unexpected K Lines array length: %d, exp: %d", len(jsonArr), KLinesArrayLen) + } + + err = json.Unmarshal(jsonArr[0], &k.Ts) + if err != nil { + return fmt.Errorf("failed to unmarshal resp index 0: %v, err: %w", string(jsonArr[0]), err) + } + + values := make([]fixedpoint.Value, len(jsonArr)-1) + for i, jsonRaw := range jsonArr[1:] { + err = json.Unmarshal(jsonRaw, &values[i]) + if err != nil { + return fmt.Errorf("failed to unmarshal resp index %d: %v, err: %w", i+1, string(jsonRaw), err) + } + } + k.Open = values[0] + k.High = values[1] + k.Low = values[2] + k.Close = values[3] + k.Volume = values[4] + k.QuoteVolume = values[5] + k.UsdtVolume = values[6] + + return nil +} + +//go:generate GetRequest -url "/api/v2/spot/market/candles" -type GetKLineRequest -responseDataType .KLineResponse +type GetKLineRequest struct { + client requestgen.APIClient + + symbol string `param:"symbol,query"` + granularity string `param:"granularity,query"` + startTime *time.Time `param:"startTime,milliseconds,query"` + endTime *time.Time `param:"endTime,milliseconds,query"` + // Limit number default 100 max 1000 + limit *string `param:"limit,query"` +} + +func (s *Client) NewGetKLineRequest() *GetKLineRequest { + return &GetKLineRequest{client: s.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_k_line_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_k_line_request_requestgen.go new file mode 100644 index 000000000..2e4157ed2 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_k_line_request_requestgen.go @@ -0,0 +1,224 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/market/candles -type GetKLineRequest -responseDataType .KLineResponse"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" + "strconv" + "time" +) + +func (g *GetKLineRequest) Symbol(symbol string) *GetKLineRequest { + g.symbol = symbol + return g +} + +func (g *GetKLineRequest) Granularity(granularity string) *GetKLineRequest { + g.granularity = granularity + return g +} + +func (g *GetKLineRequest) StartTime(startTime time.Time) *GetKLineRequest { + g.startTime = &startTime + return g +} + +func (g *GetKLineRequest) EndTime(endTime time.Time) *GetKLineRequest { + g.endTime = &endTime + return g +} + +func (g *GetKLineRequest) Limit(limit string) *GetKLineRequest { + g.limit = &limit + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetKLineRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + symbol := g.symbol + + // assign parameter of symbol + params["symbol"] = symbol + // check granularity field -> json key granularity + granularity := g.granularity + + // assign parameter of granularity + params["granularity"] = granularity + // check startTime field -> json key startTime + if g.startTime != nil { + startTime := *g.startTime + + // assign parameter of startTime + // convert time.Time to milliseconds time stamp + params["startTime"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check endTime field -> json key endTime + if g.endTime != nil { + endTime := *g.endTime + + // assign parameter of endTime + // convert time.Time to milliseconds time stamp + params["endTime"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetKLineRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetKLineRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetKLineRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetKLineRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetKLineRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetKLineRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetKLineRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetKLineRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (g *GetKLineRequest) GetPath() string { + return "/api/v2/spot/market/candles" +} + +// Do generates the request object and send the request object to the API endpoint +func (g *GetKLineRequest) Do(ctx context.Context) (KLineResponse, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + var apiURL string + + apiURL = g.GetPath() + + req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data KLineResponse + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index b9d1a1ccb..a5e0e91a7 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -6,6 +6,7 @@ import ( "math" "strconv" "strings" + "time" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" @@ -341,3 +342,26 @@ func toGlobalBalanceMap(balances []Balance) types.BalanceMap { } return bm } + +func toGlobalKLines(symbol string, interval types.Interval, kLines v2.KLineResponse) []types.KLine { + gKLines := make([]types.KLine, len(kLines)) + for i, kline := range kLines { + endTime := types.Time(kline.Ts.Time().Add(interval.Duration() - time.Millisecond)) + gKLines[i] = types.KLine{ + Exchange: types.ExchangeBitget, + Symbol: symbol, + StartTime: types.Time(kline.Ts), + EndTime: endTime, + Interval: interval, + Open: kline.Open, + Close: kline.Close, + High: kline.High, + Low: kline.Low, + Volume: kline.Volume, + QuoteVolume: kline.QuoteVolume, + // Bitget doesn't support close flag in REST API + Closed: false, + } + } + return gKLines +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 19e759aa1..8bf66e9b7 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -3,6 +3,7 @@ package bitget import ( "strconv" "testing" + "time" "github.com/stretchr/testify/assert" @@ -598,3 +599,88 @@ func Test_toGlobalBalanceMap(t *testing.T) { }, })) } + +func Test_toGlobalKLines(t *testing.T) { + symbol := "BTCUSDT" + interval := types.Interval15m + + resp := v2.KLineResponse{ + /* + [ + { + "Ts": "1699816800000", + "OpenPrice": 29045.3, + "HighPrice": 29228.56, + "LowPrice": 29045.3, + "ClosePrice": 29228.56, + "Volume": 9.265593, + "QuoteVolume": 270447.43520753, + "UsdtVolume": 270447.43520753 + }, + { + "Ts": "1699816800000", + "OpenPrice": 29167.33, + "HighPrice": 29229.08, + "LowPrice": 29000, + "ClosePrice": 29045.3, + "Volume": 9.295508, + "QuoteVolume": 270816.87513775, + "UsdtVolume": 270816.87513775 + } + ] + */ + { + Ts: types.NewMillisecondTimestampFromInt(1691486100000), + Open: fixedpoint.NewFromFloat(29045.3), + High: fixedpoint.NewFromFloat(29228.56), + Low: fixedpoint.NewFromFloat(29045.3), + Close: fixedpoint.NewFromFloat(29228.56), + Volume: fixedpoint.NewFromFloat(9.265593), + QuoteVolume: fixedpoint.NewFromFloat(270447.43520753), + UsdtVolume: fixedpoint.NewFromFloat(270447.43520753), + }, + { + Ts: types.NewMillisecondTimestampFromInt(1691487000000), + Open: fixedpoint.NewFromFloat(29167.33), + High: fixedpoint.NewFromFloat(29229.08), + Low: fixedpoint.NewFromFloat(29000), + Close: fixedpoint.NewFromFloat(29045.3), + Volume: fixedpoint.NewFromFloat(9.295508), + QuoteVolume: fixedpoint.NewFromFloat(270816.87513775), + UsdtVolume: fixedpoint.NewFromFloat(270447.43520753), + }, + } + + expKlines := []types.KLine{ + { + Exchange: types.ExchangeBitget, + Symbol: symbol, + StartTime: types.Time(resp[0].Ts.Time()), + EndTime: types.Time(resp[0].Ts.Time().Add(interval.Duration() - time.Millisecond)), + Interval: interval, + Open: fixedpoint.NewFromFloat(29045.3), + Close: fixedpoint.NewFromFloat(29228.56), + High: fixedpoint.NewFromFloat(29228.56), + Low: fixedpoint.NewFromFloat(29045.3), + Volume: fixedpoint.NewFromFloat(9.265593), + QuoteVolume: fixedpoint.NewFromFloat(270447.43520753), + Closed: false, + }, + { + Exchange: types.ExchangeBitget, + Symbol: symbol, + StartTime: types.Time(resp[1].Ts.Time()), + EndTime: types.Time(resp[1].Ts.Time().Add(interval.Duration() - time.Millisecond)), + Interval: interval, + Open: fixedpoint.NewFromFloat(29167.33), + Close: fixedpoint.NewFromFloat(29045.3), + High: fixedpoint.NewFromFloat(29229.08), + Low: fixedpoint.NewFromFloat(29000), + Volume: fixedpoint.NewFromFloat(9.295508), + QuoteVolume: fixedpoint.NewFromFloat(270816.87513775), + Closed: false, + }, + } + + assert.Equal(t, toGlobalKLines(symbol, interval, resp), expKlines) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 988b626d2..e94166773 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -21,9 +21,10 @@ const ( PlatformToken = "BGB" - queryLimit = 100 - maxOrderIdLen = 36 - queryMaxDuration = 90 * 24 * time.Hour + queryLimit = 100 + defaultKLineLimit = 100 + maxOrderIdLen = 36 + queryMaxDuration = 90 * 24 * time.Hour ) var log = logrus.WithFields(logrus.Fields{ @@ -49,6 +50,8 @@ var ( queryTradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) // cancelOrderRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/trade/Cancel-Order cancelOrderRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // kLineRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/market/Get-Candle-Data + kLineOrderRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) ) type Exchange struct { @@ -153,9 +156,56 @@ func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[str return tickers, nil } +// QueryKLines queries the k line data by interval and time range...etc. +// +// If you provide only the start time, the system will return the latest data. +// If you provide both the start and end times, the system will return data within the specified range. +// If you provide only the end time, the system will return data that occurred before the end time. +// +// The end time has different limits. 1m, 5m can query for one month,15m can query for 52 days,30m can query for 62 days, +// 1H can query for 83 days,4H can query for 240 days,6H can query for 360 days. func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions) ([]types.KLine, error) { - // TODO implement me - panic("implement me") + req := e.v2Client.NewGetKLineRequest().Symbol(symbol) + intervalStr, found := toLocalGranularity[interval] + if !found { + return nil, fmt.Errorf("%s not supported, supported granlarity: %+v", intervalStr, toLocalGranularity) + } + req.Granularity(intervalStr) + + limit := uint64(options.Limit) + if limit > defaultKLineLimit || limit <= 0 { + log.Debugf("limtit is exceeded or zero, update to %d, got: %d", defaultKLineLimit, options.Limit) + limit = defaultKLineLimit + } + req.Limit(strconv.FormatUint(limit, 10)) + + if options.StartTime != nil { + req.StartTime(*options.StartTime) + } + + if options.EndTime != nil { + if options.StartTime != nil && options.EndTime.Before(*options.StartTime) { + return nil, fmt.Errorf("end time %s before start time %s", *options.EndTime, *options.StartTime) + } + + ok, duration := hasMaxDuration(interval) + if ok && time.Since(*options.EndTime) > duration { + return nil, fmt.Errorf("end time %s are greater than max duration %s", *options.EndTime, duration) + } + req.EndTime(*options.EndTime) + } + + if err := kLineOrderRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("query klines rate limiter wait error: %w", err) + } + + resp, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call k line, err: %w", err) + } + + kLines := toGlobalKLines(symbol, interval, resp) + return types.SortKLinesAscending(kLines), nil } func (e *Exchange) QueryAccount(ctx context.Context) (*types.Account, error) { diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go index 0d5d5771a..19c87c484 100644 --- a/pkg/exchange/bitget/types.go +++ b/pkg/exchange/bitget/types.go @@ -299,8 +299,43 @@ var ( "candle1D": types.Interval1d, "candle1W": types.Interval1w, } + + // we align utc time zone + toLocalGranularity = map[types.Interval]string{ + types.Interval1m: "1min", + types.Interval5m: "5min", + types.Interval15m: "15min", + types.Interval30m: "30min", + types.Interval1h: "1h", + types.Interval4h: "4h", + types.Interval6h: "6Hutc", + types.Interval12h: "12Hutc", + types.Interval1d: "1Dutc", + types.Interval3d: "3Dutc", + types.Interval1w: "1Wutc", + types.Interval1mo: "1Mutc", + } ) +func hasMaxDuration(interval types.Interval) (bool, time.Duration) { + switch interval { + case types.Interval1m, types.Interval5m: + return true, 30 * 24 * time.Hour + case types.Interval15m: + return true, 52 * 24 * time.Hour + case types.Interval30m: + return true, 62 * 24 * time.Hour + case types.Interval1h: + return true, 83 * 24 * time.Hour + case types.Interval4h: + return true, 240 * 24 * time.Hour + case types.Interval6h: + return true, 360 * 24 * time.Hour + default: + return false, 0 * time.Duration(0) + } +} + type KLine struct { StartTime types.MillisecondTimestamp OpenPrice fixedpoint.Value diff --git a/pkg/exchange/bitget/types_test.go b/pkg/exchange/bitget/types_test.go index 185056620..13cc344a7 100644 --- a/pkg/exchange/bitget/types_test.go +++ b/pkg/exchange/bitget/types_test.go @@ -6,38 +6,35 @@ import ( "github.com/stretchr/testify/assert" - "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) -func TestKLine_ToGlobal(t *testing.T) { - startTime := int64(1698744600000) - interval := types.Interval1m - k := KLine{ - StartTime: types.NewMillisecondTimestampFromInt(startTime), - OpenPrice: fixedpoint.NewFromFloat(34361.49), - HighestPrice: fixedpoint.NewFromFloat(34458.98), - LowestPrice: fixedpoint.NewFromFloat(34355.53), - ClosePrice: fixedpoint.NewFromFloat(34416.41), - Volume: fixedpoint.NewFromFloat(99.6631), - } +func Test_hasMaxDuration(t *testing.T) { + ok, duration := hasMaxDuration(types.Interval1m) + assert.True(t, ok) + assert.Equal(t, 30*24*time.Hour, duration) - assert.Equal(t, types.KLine{ - Exchange: types.ExchangeBitget, - Symbol: "BTCUSDT", - StartTime: types.Time(types.NewMillisecondTimestampFromInt(startTime).Time()), - EndTime: types.Time(types.NewMillisecondTimestampFromInt(startTime).Time().Add(interval.Duration() - time.Millisecond)), - Interval: interval, - Open: fixedpoint.NewFromFloat(34361.49), - Close: fixedpoint.NewFromFloat(34416.41), - High: fixedpoint.NewFromFloat(34458.98), - Low: fixedpoint.NewFromFloat(34355.53), - Volume: fixedpoint.NewFromFloat(99.6631), - QuoteVolume: fixedpoint.Zero, - TakerBuyBaseAssetVolume: fixedpoint.Zero, - TakerBuyQuoteAssetVolume: fixedpoint.Zero, - LastTradeID: 0, - NumberOfTrades: 0, - Closed: false, - }, k.ToGlobal(interval, "BTCUSDT")) + ok, duration = hasMaxDuration(types.Interval5m) + assert.True(t, ok) + assert.Equal(t, 30*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval15m) + assert.True(t, ok) + assert.Equal(t, 52*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval30m) + assert.True(t, ok) + assert.Equal(t, 62*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval1h) + assert.True(t, ok) + assert.Equal(t, 83*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval4h) + assert.True(t, ok) + assert.Equal(t, 240*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval6h) + assert.True(t, ok) + assert.Equal(t, 360*24*time.Hour, duration) } From eb04eaeea44ee3ac27cb0cc6f303d3b5859d63dd Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 13 Nov 2023 12:51:19 +0800 Subject: [PATCH 2/2] pkg/exchange: types.kline end time should -1 time.Millisecond --- pkg/exchange/bitget/convert.go | 2 ++ pkg/exchange/bybit/convert.go | 2 +- pkg/exchange/bybit/convert_test.go | 4 ++-- pkg/types/kline.go | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index a5e0e91a7..6c3c92546 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -346,6 +346,8 @@ func toGlobalBalanceMap(balances []Balance) types.BalanceMap { func toGlobalKLines(symbol string, interval types.Interval, kLines v2.KLineResponse) []types.KLine { gKLines := make([]types.KLine, len(kLines)) for i, kline := range kLines { + // follow the binance rule, to avoid endTime overlapping with the next startTime. So we subtract -1 time.Millisecond + // on endTime. endTime := types.Time(kline.Ts.Time().Add(interval.Duration() - time.Millisecond)) gKLines[i] = types.KLine{ Exchange: types.ExchangeBitget, diff --git a/pkg/exchange/bybit/convert.go b/pkg/exchange/bybit/convert.go index d89f194fe..8c2d91541 100644 --- a/pkg/exchange/bybit/convert.go +++ b/pkg/exchange/bybit/convert.go @@ -368,7 +368,7 @@ func toLocalInterval(interval types.Interval) (string, error) { func toGlobalKLines(symbol string, interval types.Interval, klines []bybitapi.KLine) []types.KLine { gKLines := make([]types.KLine, len(klines)) for i, kline := range klines { - endTime := types.Time(kline.StartTime.Time().Add(interval.Duration())) + endTime := types.Time(kline.StartTime.Time().Add(interval.Duration() - time.Millisecond)) gKLines[i] = types.KLine{ Exchange: types.ExchangeBybit, Symbol: symbol, diff --git a/pkg/exchange/bybit/convert_test.go b/pkg/exchange/bybit/convert_test.go index a5ddb08bc..daac379e3 100644 --- a/pkg/exchange/bybit/convert_test.go +++ b/pkg/exchange/bybit/convert_test.go @@ -836,7 +836,7 @@ func Test_toGlobalKLines(t *testing.T) { Exchange: types.ExchangeBybit, Symbol: resp.Symbol, StartTime: types.Time(resp.List[0].StartTime.Time()), - EndTime: types.Time(resp.List[0].StartTime.Time().Add(interval.Duration())), + EndTime: types.Time(resp.List[0].StartTime.Time().Add(interval.Duration() - time.Millisecond)), Interval: interval, Open: fixedpoint.NewFromFloat(29045.3), Close: fixedpoint.NewFromFloat(29228.56), @@ -850,7 +850,7 @@ func Test_toGlobalKLines(t *testing.T) { Exchange: types.ExchangeBybit, Symbol: resp.Symbol, StartTime: types.Time(resp.List[1].StartTime.Time()), - EndTime: types.Time(resp.List[1].StartTime.Time().Add(interval.Duration())), + EndTime: types.Time(resp.List[1].StartTime.Time().Add(interval.Duration() - time.Millisecond)), Interval: interval, Open: fixedpoint.NewFromFloat(29167.33), Close: fixedpoint.NewFromFloat(29045.3), diff --git a/pkg/types/kline.go b/pkg/types/kline.go index 6bf863c30..bd31ed9a2 100644 --- a/pkg/types/kline.go +++ b/pkg/types/kline.go @@ -54,7 +54,9 @@ type KLine struct { Symbol string `json:"symbol" db:"symbol"` StartTime Time `json:"startTime" db:"start_time"` - EndTime Time `json:"endTime" db:"end_time"` + // EndTime follows the binance rule, to avoid endTime overlapping with the next startTime. So if your end time (2023-01-01 01:00:00) + // are overlapping with next start time interval (2023-01-01 01:00:00), you should subtract -1 time.millisecond on EndTime. + EndTime Time `json:"endTime" db:"end_time"` Interval Interval `json:"interval" db:"interval"`