From ef8d1c70461ffc1bec64cd914b5110d1e93ce870 Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 24 Jul 2023 23:27:43 +0800 Subject: [PATCH 1/2] pkg/exchange: support QueryTickers API on bybit --- pkg/exchange/bybit/bybitapi/client.go | 11 +- pkg/exchange/bybit/bybitapi/client_test.go | 12 ++ .../bybit/bybitapi/get_tickers_request.go | 72 ++++++++ .../get_tickers_request_requestgen.go | 172 ++++++++++++++++++ pkg/exchange/bybit/convert.go | 36 ++-- pkg/exchange/bybit/convert_test.go | 70 +++++++ pkg/exchange/bybit/exchange.go | 69 +++++++ 7 files changed, 415 insertions(+), 27 deletions(-) create mode 100644 pkg/exchange/bybit/bybitapi/get_tickers_request.go create mode 100644 pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go diff --git a/pkg/exchange/bybit/bybitapi/client.go b/pkg/exchange/bybit/bybitapi/client.go index aed96e419..cfd8ac2bb 100644 --- a/pkg/exchange/bybit/bybitapi/client.go +++ b/pkg/exchange/bybit/bybitapi/client.go @@ -157,9 +157,10 @@ sample: */ type APIResponse struct { - RetCode uint `json:"retCode"` - RetMsg string `json:"retMsg"` - Result json.RawMessage `json:"result"` - RetExtInfo json.RawMessage `json:"retExtInfo"` - Time types.MillisecondTimestamp `json:"time"` + RetCode uint `json:"retCode"` + RetMsg string `json:"retMsg"` + Result json.RawMessage `json:"result"` + RetExtInfo json.RawMessage `json:"retExtInfo"` + // Time is current timestamp (ms) + Time types.MillisecondTimestamp `json:"time"` } diff --git a/pkg/exchange/bybit/bybitapi/client_test.go b/pkg/exchange/bybit/bybitapi/client_test.go index bc89d60fc..e18e78941 100644 --- a/pkg/exchange/bybit/bybitapi/client_test.go +++ b/pkg/exchange/bybit/bybitapi/client_test.go @@ -45,4 +45,16 @@ func TestClient(t *testing.T) { assert.NoError(t, err) t.Logf("instrumentsInfo: %+v", instrumentsInfo) }) + + t.Run("GetTicker", func(t *testing.T) { + req := client.NewGetTickersRequest() + apiResp, err := req.Symbol("BTCUSDT").Do(ctx) + assert.NoError(t, err) + t.Logf("apiResp: %+v", apiResp) + + req = client.NewGetTickersRequest() + tickers, err := req.Symbol("BTCUSDT").DoWithResponseTime(ctx) + assert.NoError(t, err) + t.Logf("tickers: %+v", tickers) + }) } diff --git a/pkg/exchange/bybit/bybitapi/get_tickers_request.go b/pkg/exchange/bybit/bybitapi/get_tickers_request.go new file mode 100644 index 000000000..2fa79a38f --- /dev/null +++ b/pkg/exchange/bybit/bybitapi/get_tickers_request.go @@ -0,0 +1,72 @@ +package bybitapi + +import ( + "context" + "encoding/json" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/requestgen" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Result +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Result + +type Tickers struct { + Category Category `json:"category"` + List []Ticker `json:"list"` + + // ClosedTime is current timestamp (ms). This value is obtained from outside APIResponse. + ClosedTime types.MillisecondTimestamp +} + +type Ticker struct { + Symbol string `json:"symbol"` + Bid1Price fixedpoint.Value `json:"bid1Price"` + Bid1Size fixedpoint.Value `json:"bid1Size"` + Ask1Price fixedpoint.Value `json:"ask1Price"` + Ask1Size fixedpoint.Value `json:"ask1Size"` + LastPrice fixedpoint.Value `json:"lastPrice"` + PrevPrice24H fixedpoint.Value `json:"prevPrice24h"` + Price24HPcnt fixedpoint.Value `json:"price24hPcnt"` + HighPrice24H fixedpoint.Value `json:"highPrice24h"` + LowPrice24H fixedpoint.Value `json:"lowPrice24h"` + Turnover24H fixedpoint.Value `json:"turnover24h"` + Volume24H fixedpoint.Value `json:"volume24h"` + UsdIndexPrice fixedpoint.Value `json:"usdIndexPrice"` +} + +// GetTickersRequest without **-responseDataType .InstrumentsInfo** in generation command, because the caller +// needs the APIResponse.Time. We implemented the DoWithResponseTime to handle this. +// +//go:generate GetRequest -url "/v5/market/tickers" -type GetTickersRequest +type GetTickersRequest struct { + client requestgen.APIClient + + category Category `param:"category,query" validValues:"spot"` + symbol *string `param:"symbol,query"` +} + +func (c *RestClient) NewGetTickersRequest() *GetTickersRequest { + return &GetTickersRequest{ + client: c, + category: CategorySpot, + } +} + +func (g *GetTickersRequest) DoWithResponseTime(ctx context.Context) (*Tickers, error) { + resp, err := g.Do(ctx) + if err != nil { + return nil, err + } + + var data Tickers + if err := json.Unmarshal(resp.Result, &data); err != nil { + return nil, err + } + + // Our types.Ticker requires the closed time, but this API does not provide it. This API returns the Tickers of the + // past 24 hours, so in terms of closed time, it is the current time, so fill it in Tickers.ClosedTime. + data.ClosedTime = resp.Time + return &data, nil +} diff --git a/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go new file mode 100644 index 000000000..0f85973d8 --- /dev/null +++ b/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go @@ -0,0 +1,172 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Result -url /v5/market/tickers -type GetTickersRequest"; DO NOT EDIT. + +package bybitapi + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" +) + +func (g *GetTickersRequest) Category(category Category) *GetTickersRequest { + g.category = category + return g +} + +func (g *GetTickersRequest) Symbol(symbol string) *GetTickersRequest { + g.symbol = &symbol + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetTickersRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check category field -> json key category + category := g.category + + // TEMPLATE check-valid-values + switch category { + case "spot": + params["category"] = category + + default: + return nil, fmt.Errorf("category value %v is invalid", category) + + } + // END TEMPLATE check-valid-values + + // assign parameter of category + params["category"] = category + // check symbol field -> json key symbol + if g.symbol != nil { + symbol := *g.symbol + + // assign parameter of symbol + params["symbol"] = symbol + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetTickersRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetTickersRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetTickersRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetTickersRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetTickersRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetTickersRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetTickersRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetTickersRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (g *GetTickersRequest) Do(ctx context.Context) (*APIResponse, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + apiURL := "/v5/market/tickers" + + req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + return &apiResponse, nil +} diff --git a/pkg/exchange/bybit/convert.go b/pkg/exchange/bybit/convert.go index 0b37d7adc..e83f31d31 100644 --- a/pkg/exchange/bybit/convert.go +++ b/pkg/exchange/bybit/convert.go @@ -2,34 +2,13 @@ package bybit import ( "math" + "time" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" "github.com/c9s/bbgo/pkg/types" ) func toGlobalMarket(m bybitapi.Instrument) types.Market { - // sample: - //Symbol: BTCUSDT - //BaseCoin: BTC - //QuoteCoin: USDT - //Innovation: 0 - //Status: Trading - //MarginTrading: both - // - //LotSizeFilter: - //{ - // BasePrecision: 0.000001 - // QuotePrecision: 0.00000001 - // MinOrderQty: 0.000048 - // MaxOrderQty: 71.73956243 - // MinOrderAmt: 1 - // MaxOrderAmt: 2000000 - //} - // - //PriceFilter: - //{ - // TickSize: 0.01 - //} return types.Market{ Symbol: m.Symbol, LocalSymbol: m.Symbol, @@ -51,3 +30,16 @@ func toGlobalMarket(m bybitapi.Instrument) types.Market { TickSize: m.PriceFilter.TickSize, } } + +func toGlobalTicker(stats bybitapi.Ticker, time time.Time) types.Ticker { + return types.Ticker{ + Volume: stats.Volume24H, + Last: stats.LastPrice, + Open: stats.PrevPrice24H, // Market price 24 hours ago + High: stats.HighPrice24H, + Low: stats.LowPrice24H, + Buy: stats.Bid1Price, + Sell: stats.Ask1Price, + Time: time, + } +} diff --git a/pkg/exchange/bybit/convert_test.go b/pkg/exchange/bybit/convert_test.go index 22325b953..6ec60cfe0 100644 --- a/pkg/exchange/bybit/convert_test.go +++ b/pkg/exchange/bybit/convert_test.go @@ -3,6 +3,7 @@ package bybit import ( "math" "testing" + "time" "github.com/stretchr/testify/assert" @@ -12,6 +13,26 @@ import ( ) func TestToGlobalMarket(t *testing.T) { + // sample: + //{ + // "Symbol": "BTCUSDT", + // "BaseCoin": "BTC", + // "QuoteCoin": "USDT", + // "Innovation": 0, + // "Status": "Trading", + // "MarginTrading": "both", + // "LotSizeFilter": { + // "BasePrecision": 0.000001, + // "QuotePrecision": 0.00000001, + // "MinOrderQty": 0.000048, + // "MaxOrderQty": 71.73956243, + // "MinOrderAmt": 1, + // "MaxOrderAmt": 2000000 + // }, + // "PriceFilter": { + // "TickSize": 0.01 + // } + //} inst := bybitapi.Instrument{ Symbol: "BTCUSDT", BaseCoin: "BTC", @@ -60,3 +81,52 @@ func TestToGlobalMarket(t *testing.T) { assert.Equal(t, toGlobalMarket(inst), exp) } + +func TestToGlobalTicker(t *testing.T) { + // sample + //{ + // "symbol": "BTCUSDT", + // "bid1Price": "28995.98", + // "bid1Size": "4.741552", + // "ask1Price": "28995.99", + // "ask1Size": "0.16075", + // "lastPrice": "28994", + // "prevPrice24h": "29900", + // "price24hPcnt": "-0.0303", + // "highPrice24h": "30344.78", + // "lowPrice24h": "28948.87", + // "turnover24h": "184705500.13172874", + // "volume24h": "6240.807096", + // "usdIndexPrice": "28977.82001643" + //} + ticker := bybitapi.Ticker{ + Symbol: "BTCUSDT", + Bid1Price: fixedpoint.NewFromFloat(28995.98), + Bid1Size: fixedpoint.NewFromFloat(4.741552), + Ask1Price: fixedpoint.NewFromFloat(28995.99), + Ask1Size: fixedpoint.NewFromFloat(0.16075), + LastPrice: fixedpoint.NewFromFloat(28994), + PrevPrice24H: fixedpoint.NewFromFloat(29900), + Price24HPcnt: fixedpoint.NewFromFloat(-0.0303), + HighPrice24H: fixedpoint.NewFromFloat(30344.78), + LowPrice24H: fixedpoint.NewFromFloat(28948.87), + Turnover24H: fixedpoint.NewFromFloat(184705500.13172874), + Volume24H: fixedpoint.NewFromFloat(6240.807096), + UsdIndexPrice: fixedpoint.NewFromFloat(28977.82001643), + } + + timeNow := time.Now() + + exp := types.Ticker{ + Time: timeNow, + Volume: ticker.Volume24H, + Last: ticker.LastPrice, + Open: ticker.PrevPrice24H, + High: ticker.HighPrice24H, + Low: ticker.LowPrice24H, + Buy: ticker.Bid1Price, + Sell: ticker.Ask1Price, + } + + assert.Equal(t, toGlobalTicker(ticker, timeNow), exp) +} diff --git a/pkg/exchange/bybit/exchange.go b/pkg/exchange/bybit/exchange.go index 5abd5c645..9d8a01059 100644 --- a/pkg/exchange/bybit/exchange.go +++ b/pkg/exchange/bybit/exchange.go @@ -2,13 +2,23 @@ package bybit import ( "context" + "fmt" + "time" "github.com/sirupsen/logrus" + "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" "github.com/c9s/bbgo/pkg/types" ) +// https://bybit-exchange.github.io/docs/zh-TW/v5/rate-limit +// sharedRateLimiter indicates that the API belongs to the public API. +// +// The default order limiter apply 2 requests per second and a 2 initial bucket +// this includes QueryMarkets, QueryTicker +var sharedRateLimiter = rate.NewLimiter(rate.Every(time.Second/2), 2) + var log = logrus.WithFields(logrus.Fields{ "exchange": "bybit", }) @@ -47,8 +57,14 @@ func (e *Exchange) PlatformFeeCurrency() string { } func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { + if err := sharedRateLimiter.Wait(ctx); err != nil { + log.WithError(err).Errorf("markets rate limiter wait error") + return nil, err + } + instruments, err := e.client.NewGetInstrumentsInfoRequest().Do(ctx) if err != nil { + log.Warnf("failed to query instruments, err: %v", err) return nil, err } @@ -59,3 +75,56 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { return marketMap, nil } + +func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticker, error) { + if err := sharedRateLimiter.Wait(ctx); err != nil { + log.WithError(err).Errorf("ticker rate limiter wait error") + return nil, err + } + + s, err := e.client.NewGetTickersRequest().Symbol(symbol).DoWithResponseTime(ctx) + if err != nil { + log.Warnf("failed to get tickers, symbol: %s, err: %v", symbol, err) + return nil, err + } + + if len(s.List) != 1 { + log.Warnf("unexpected ticker length, exp: 1, got: %d", len(s.List)) + return nil, fmt.Errorf("unexpected ticker lenght, exp:1, got:%d", len(s.List)) + } + + ticker := toGlobalTicker(s.List[0], s.ClosedTime.Time()) + return &ticker, nil +} + +func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[string]types.Ticker, error) { + tickers := map[string]types.Ticker{} + if len(symbols) > 0 { + for _, s := range symbols { + t, err := e.QueryTicker(ctx, s) + if err != nil { + return nil, err + } + + tickers[s] = *t + } + + return tickers, nil + } + + if err := sharedRateLimiter.Wait(ctx); err != nil { + log.WithError(err).Errorf("ticker rate limiter wait error") + return nil, err + } + allTickers, err := e.client.NewGetTickersRequest().DoWithResponseTime(ctx) + if err != nil { + log.Warnf("failed to get tickers, err: %v", err) + return nil, err + } + + for _, s := range allTickers.List { + tickers[s.Symbol] = toGlobalTicker(s, allTickers.ClosedTime.Time()) + } + + return tickers, nil +} From b71030c5db033df2e394514d2b01b7e8ec4faa90 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 25 Jul 2023 15:09:57 +0800 Subject: [PATCH 2/2] pkg: return err if rate limit err --- pkg/exchange/binance/cancel_replace.go | 1 + pkg/exchange/binance/exchange.go | 9 ++++++--- pkg/interact/telegram.go | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/exchange/binance/cancel_replace.go b/pkg/exchange/binance/cancel_replace.go index c993c66d0..e98090a48 100644 --- a/pkg/exchange/binance/cancel_replace.go +++ b/pkg/exchange/binance/cancel_replace.go @@ -12,6 +12,7 @@ import ( func (e *Exchange) CancelReplace(ctx context.Context, cancelReplaceMode types.CancelReplaceModeType, o types.Order) (*types.Order, error) { if err := orderLimiter.Wait(ctx); err != nil { log.WithError(err).Errorf("order rate limiter wait error") + return nil, err } if e.IsFutures || e.IsMargin { diff --git a/pkg/exchange/binance/exchange.go b/pkg/exchange/binance/exchange.go index 43d7554af..9f8534fc8 100644 --- a/pkg/exchange/binance/exchange.go +++ b/pkg/exchange/binance/exchange.go @@ -764,8 +764,9 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, } */ - if err := orderLimiter.Wait(ctx); err != nil { + if err = orderLimiter.Wait(ctx); err != nil { log.WithError(err).Errorf("order rate limiter wait error") + return nil, err } log.Infof("querying closed orders %s from %s <=> %s ...", symbol, since, until) @@ -822,8 +823,9 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, } func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) (err error) { - if err := orderLimiter.Wait(ctx); err != nil { + if err = orderLimiter.Wait(ctx); err != nil { log.WithError(err).Errorf("order rate limiter wait error") + return err } if e.IsFutures { @@ -1086,8 +1088,9 @@ func (e *Exchange) submitSpotOrder(ctx context.Context, order types.SubmitOrder) } func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (createdOrder *types.Order, err error) { - if err := orderLimiter.Wait(ctx); err != nil { + if err = orderLimiter.Wait(ctx); err != nil { log.WithError(err).Errorf("order rate limiter wait error") + return nil, err } if e.IsMargin { diff --git a/pkg/interact/telegram.go b/pkg/interact/telegram.go index c2a45a002..4635cbe39 100644 --- a/pkg/interact/telegram.go +++ b/pkg/interact/telegram.go @@ -73,6 +73,7 @@ func (r *TelegramReply) Send(message string) { for _, split := range splits { if err := sendLimiter.Wait(ctx); err != nil { log.WithError(err).Errorf("telegram send limit exceeded") + return } checkSendErr(r.bot.Send(r.session.Chat, split)) } @@ -175,6 +176,7 @@ func (tm *Telegram) Start(ctx context.Context) { for i, split := range splits { if err := sendLimiter.Wait(ctx); err != nil { log.WithError(err).Errorf("telegram send limit exceeded") + return } if i == len(splits)-1 { // only set menu on the last message