From ffea4901edd05c6174e232aaf6ad1a7c089046bc Mon Sep 17 00:00:00 2001 From: narumi Date: Fri, 3 Nov 2023 15:07:24 +0800 Subject: [PATCH 01/33] fix buy quantity --- pkg/strategy/rebalance/strategy.go | 45 +++++++++++++++++------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/pkg/strategy/rebalance/strategy.go b/pkg/strategy/rebalance/strategy.go index 22896d24b..40c0fa821 100644 --- a/pkg/strategy/rebalance/strategy.go +++ b/pkg/strategy/rebalance/strategy.go @@ -40,7 +40,6 @@ type Strategy struct { DryRun bool `json:"dryRun"` OnStart bool `json:"onStart"` // rebalance on start - session *bbgo.ExchangeSession symbols []string markets map[string]types.Market activeOrderBook *bbgo.ActiveOrderBook @@ -97,11 +96,9 @@ func (s *Strategy) Validate() error { func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) {} func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.session = session - s.markets = make(map[string]types.Market) for _, symbol := range s.symbols { - market, ok := s.session.Market(symbol) + market, ok := session.Market(symbol) if !ok { return fmt.Errorf("market %s not found", symbol) } @@ -112,7 +109,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.MultiMarketStrategy.Initialize(ctx, s.Environment, session, s.markets, ID) s.activeOrderBook = bbgo.NewActiveOrderBook("") - s.activeOrderBook.BindStream(s.session.UserDataStream) + s.activeOrderBook.BindStream(session.UserDataStream) session.UserDataStream.OnStart(func() { if s.OnStart { @@ -137,7 +134,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. func (s *Strategy) rebalance(ctx context.Context) { // cancel active orders before rebalance - if err := s.session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { + if err := s.Session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { log.WithError(err).Errorf("failed to cancel orders") } @@ -174,7 +171,7 @@ func (s *Strategy) queryMidPrices(ctx context.Context) (types.ValueMap, error) { continue } - ticker, err := s.session.Exchange.QueryTicker(ctx, currency+s.QuoteCurrency) + ticker, err := s.Session.Exchange.QueryTicker(ctx, currency+s.QuoteCurrency) if err != nil { return nil, err } @@ -186,7 +183,7 @@ func (s *Strategy) queryMidPrices(ctx context.Context) (types.ValueMap, error) { func (s *Strategy) selectBalances() (types.BalanceMap, error) { m := make(types.BalanceMap) - balances := s.session.GetAccount().Balances() + balances := s.Session.GetAccount().Balances() for currency := range s.TargetWeights { balance, ok := balances[currency] if !ok { @@ -235,28 +232,36 @@ func (s *Strategy) generateOrder(ctx context.Context) (*types.SubmitOrder, error quantity = quantity.Abs() } - if s.MaxAmount.Float64() > 0 { - quantity = bbgo.AdjustQuantityByMaxAmount(quantity, midPrice, s.MaxAmount) - log.Infof("adjust quantity %s (%s %s @ %s) by max amount %s", - quantity.String(), - symbol, - side.String(), - midPrice.String(), - s.MaxAmount.String()) + ticker, err := s.Session.Exchange.QueryTicker(ctx, symbol) + if err != nil { + return nil, err } + var price fixedpoint.Value if side == types.SideTypeBuy { - quantity = fixedpoint.Min(quantity, balances[s.QuoteCurrency].Available.Div(midPrice)) + price = ticker.Buy + quantity = fixedpoint.Min(quantity, balances[s.QuoteCurrency].Available.Div(ticker.Sell)) } else if side == types.SideTypeSell { + price = ticker.Sell quantity = fixedpoint.Min(quantity, balances[market.BaseCurrency].Available) } - if market.IsDustQuantity(quantity, midPrice) { + if s.MaxAmount.Float64() > 0 { + quantity = bbgo.AdjustQuantityByMaxAmount(quantity, price, s.MaxAmount) + log.Infof("adjusted quantity %s (%s %s @ %s) by max amount %s", + quantity.String(), + symbol, + side.String(), + price.String(), + s.MaxAmount.String()) + } + + if market.IsDustQuantity(quantity, price) { log.Infof("quantity %s (%s %s @ %s) is dust quantity, skip", quantity.String(), symbol, side.String(), - midPrice.String()) + price.String()) continue } @@ -265,7 +270,7 @@ func (s *Strategy) generateOrder(ctx context.Context) (*types.SubmitOrder, error Side: side, Type: s.OrderType, Quantity: quantity, - Price: midPrice, + Price: price, }, nil } return nil, nil From cdebcc9a587eaf7bae49a1ff5aa8b22ec2278181 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 4 Nov 2023 12:55:22 +0800 Subject: [PATCH 02/33] add .env.local.example --- .env.local.example | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .env.local.example diff --git a/.env.local.example b/.env.local.example new file mode 100644 index 000000000..b6552d4d7 --- /dev/null +++ b/.env.local.example @@ -0,0 +1,13 @@ +SLACK_TOKEN=YOUR_TOKEN +SLACK_CHANNEL=CHANNEL_NAME + +# DB_DRIVER="sqlite3" +# DB_DSN="bbgo.sqlite3" +DB_DRIVER=mysql +DB_DSN=root@tcp(127.0.0.1:3306)/bbgo + +MAX_API_KEY=YOUR_API_KEY +MAX_API_SECRET=YOUR_API_SECRET + +BINANCE_API_KEY=YOUR_API_KEY +BINANCE_API_SECRET=YOUR_API_SECRET From 6cce5a2268b80d18e1db4262b80c4fc80e046060 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 3 Nov 2023 18:12:42 +0800 Subject: [PATCH 03/33] grid2: respect s.BaseGridNum and add a failing test case --- pkg/strategy/grid2/strategy.go | 22 +++++- pkg/strategy/grid2/strategy_test.go | 113 +++++++++++++++++++++++----- 2 files changed, 114 insertions(+), 21 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index ce3f77dc2..0c8248d93 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -796,6 +796,8 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( if numberOfSellOrders > 0 { numberOfSellOrders-- } + + s.logger.Infof("calculated number of sell orders: %d", numberOfSellOrders) } // if the maxBaseQuantity is less than minQuantity, then we need to reduce the number of the sell orders @@ -810,8 +812,12 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( s.Market.MinQuantity) if baseQuantity.Compare(minBaseQuantity) <= 0 { + s.logger.Infof("base quantity %s is less than min base quantity: %s, adjusting...", baseQuantity.String(), minBaseQuantity.String()) + baseQuantity = s.Market.RoundUpQuantityByPrecision(minBaseQuantity) numberOfSellOrders = int(math.Floor(baseInvestment.Div(baseQuantity).Float64())) + + s.logger.Infof("adjusted base quantity to %s", baseQuantity.String()) } s.logger.Infof("grid base investment sell orders: %d", numberOfSellOrders) @@ -824,7 +830,8 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( // quoteInvestment = (p1 + p2 + p3) * q // maxBuyQuantity = quoteInvestment / (p1 + p2 + p3) si := -1 - for i := len(pins) - 1 - numberOfSellOrders; i >= 0; i-- { + end := len(pins) - 1 + for i := end - numberOfSellOrders - 1; i >= 0; i-- { pin := pins[i] price := fixedpoint.Value(pin) @@ -844,6 +851,7 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( // requiredQuote = requiredQuote.Add(quantity.Mul(nextLowerPrice)) totalQuotePrice = totalQuotePrice.Add(nextLowerPrice) } + } else { // for orders that buy if s.ProfitSpread.IsZero() && i+1 == si { @@ -851,7 +859,7 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( } // should never place a buy order at the upper price - if i == len(pins)-1 { + if i == end { continue } @@ -859,8 +867,11 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( } } + s.logger.Infof("total quote price: %f", totalQuotePrice.Float64()) if totalQuotePrice.Sign() > 0 && quoteInvestment.Sign() > 0 { quoteSideQuantity := quoteInvestment.Div(totalQuotePrice) + + s.logger.Infof("quote side quantity: %f = %f / %f", quoteSideQuantity.Float64(), quoteInvestment.Float64(), totalQuotePrice.Float64()) if numberOfSellOrders > 0 { return fixedpoint.Min(quoteSideQuantity, baseQuantity), nil } @@ -1058,6 +1069,11 @@ func (s *Strategy) openGrid(ctx context.Context, session *bbgo.ExchangeSession) return err2 } + if s.BaseGridNum > 0 { + sell1 := fixedpoint.Value(s.grid.Pins[len(s.grid.Pins)-1-s.BaseGridNum]) + lastPrice = sell1.Sub(s.Market.TickSize) + } + // check if base and quote are enough var totalBase = fixedpoint.Zero var totalQuote = fixedpoint.Zero @@ -1432,6 +1448,8 @@ func calculateMinimalQuoteInvestment(market types.Market, grid *Grid) fixedpoint for i := len(pins) - 2; i >= 0; i-- { pin := pins[i] price := fixedpoint.Value(pin) + + // TODO: should we round the quote here before adding? totalQuote = totalQuote.Add(price.Mul(minQuantity)) } diff --git a/pkg/strategy/grid2/strategy_test.go b/pkg/strategy/grid2/strategy_test.go index 9f592959f..0110782dd 100644 --- a/pkg/strategy/grid2/strategy_test.go +++ b/pkg/strategy/grid2/strategy_test.go @@ -204,6 +204,65 @@ func TestStrategy_generateGridOrders(t *testing.T) { }, orders) }) + t.Run("base and quote #2", func(t *testing.T) { + gridNum := int64(22) + upperPrice := number(35500.000000) + lowerPrice := number(34450.000000) + quoteInvestment := number(18.47) + baseInvestment := number(0.010700) + lastPrice := number(34522.930000) + baseGridNum := int(20) + + s := newTestStrategy() + s.GridNum = gridNum + s.BaseGridNum = baseGridNum + s.LowerPrice = lowerPrice + s.UpperPrice = upperPrice + s.grid = NewGrid(lowerPrice, upperPrice, fixedpoint.NewFromInt(s.GridNum), s.Market.TickSize) + s.grid.CalculateArithmeticPins() + assert.Equal(t, 22, len(s.grid.Pins)) + + quantity, err := s.calculateBaseQuoteInvestmentQuantity(quoteInvestment, baseInvestment, lastPrice, s.grid.Pins) + assert.NoError(t, err) + assert.Equal(t, "0.000535", quantity.String()) + + s.QuantityOrAmount.Quantity = quantity + + orders, err := s.generateGridOrders(quoteInvestment, baseInvestment, lastPrice) + assert.NoError(t, err) + if !assert.Equal(t, 21, len(orders)) { + for _, o := range orders { + t.Logf("- %s %s", o.Price.String(), o.Side) + } + } + + assertPriceSide(t, []PriceSideAssert{ + {number(35500.0), types.SideTypeSell}, + {number(35450.0), types.SideTypeSell}, + {number(35400.0), types.SideTypeSell}, + {number(35350.0), types.SideTypeSell}, + {number(35300.0), types.SideTypeSell}, + {number(35250.0), types.SideTypeSell}, + {number(35200.0), types.SideTypeSell}, + {number(35150.0), types.SideTypeSell}, + {number(35100.0), types.SideTypeSell}, + {number(35050.0), types.SideTypeSell}, + {number(35000.0), types.SideTypeSell}, + {number(34950.0), types.SideTypeSell}, + {number(34900.0), types.SideTypeSell}, + {number(34850.0), types.SideTypeSell}, + {number(34800.0), types.SideTypeSell}, + {number(34750.0), types.SideTypeSell}, + {number(34700.0), types.SideTypeSell}, + {number(34650.0), types.SideTypeSell}, + {number(34600.0), types.SideTypeSell}, + {number(34550.0), types.SideTypeSell}, + // -- fake trade price at 34549.9 + // -- 34500 should be empty + {number(34450.0), types.SideTypeBuy}, + }, orders) + }) + t.Run("base and quote with pre-calculated baseGridNumber", func(t *testing.T) { s := newTestStrategy() s.grid = NewGrid(s.LowerPrice, s.UpperPrice, fixedpoint.NewFromInt(s.GridNum), s.Market.TickSize) @@ -519,11 +578,11 @@ func newTestMarket(symbol string) types.Market { BaseCurrency: "BTC", QuoteCurrency: "USDT", TickSize: number(0.01), - StepSize: number(0.00001), + StepSize: number(0.000001), PricePrecision: 2, VolumePrecision: 8, - MinNotional: number(10.0), - MinQuantity: number(0.001), + MinNotional: number(8.0), + MinQuantity: number(0.0003), } case "ETHUSDT": return types.Market{ @@ -534,7 +593,7 @@ func newTestMarket(symbol string) types.Market { PricePrecision: 2, VolumePrecision: 6, MinNotional: number(8.000), - MinQuantity: number(0.00030), + MinQuantity: number(0.0046), } } @@ -577,12 +636,17 @@ func newTestOrder(price, quantity fixedpoint.Value, side types.SideType) types.O } } -func newTestStrategy() *Strategy { - market := newTestMarket("BTCUSDT") +func newTestStrategy(va ...string) *Strategy { + symbol := "BTCUSDT" + if len(va) > 0 { + symbol = va[0] + } + + market := newTestMarket(symbol) s := &Strategy{ logger: logrus.NewEntry(logrus.New()), - Symbol: "BTCUSDT", + Symbol: symbol, Market: market, GridProfitStats: newGridProfitStats(market), UpperPrice: number(20_000), @@ -790,7 +854,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { } orderExecutor := gridmocks.NewMockOrderExecutor(mockCtrl) - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder, order), "%+v is not equal to %+v", order, expectedSubmitOrder) return []types.Order{ {SubmitOrder: expectedSubmitOrder}, @@ -858,7 +924,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { } orderExecutor := gridmocks.NewMockOrderExecutor(mockCtrl) - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder, order), "%+v is not equal to %+v", order, expectedSubmitOrder) return []types.Order{ {SubmitOrder: expectedSubmitOrder}, @@ -946,7 +1014,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { Market: s.Market, Tag: orderTag, } - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder, order), "%+v is not equal to %+v", order, expectedSubmitOrder) return []types.Order{ {SubmitOrder: expectedSubmitOrder}, @@ -963,7 +1033,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { Market: s.Market, Tag: orderTag, } - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder2, order), "%+v is not equal to %+v", order, expectedSubmitOrder2) return []types.Order{ {SubmitOrder: expectedSubmitOrder2}, @@ -1060,7 +1132,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { } orderExecutor := gridmocks.NewMockOrderExecutor(mockCtrl) - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder, order), "%+v is not equal to %+v", order, expectedSubmitOrder) return []types.Order{ {SubmitOrder: expectedSubmitOrder}, @@ -1078,7 +1152,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { Tag: orderTag, } - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder2, order), "%+v is not equal to %+v", order, expectedSubmitOrder2) return []types.Order{ {SubmitOrder: expectedSubmitOrder2}, @@ -1190,14 +1266,14 @@ func TestStrategy_aggregateOrderQuoteAmountAndFeeRetry(t *testing.T) { func TestStrategy_checkMinimalQuoteInvestment(t *testing.T) { t.Run("7 grids", func(t *testing.T) { - s := newTestStrategy() + s := newTestStrategy("ETHUSDT") s.UpperPrice = number(1660) s.LowerPrice = number(1630) s.QuoteInvestment = number(61) s.GridNum = 7 grid := s.newGrid() minQuoteInvestment := calculateMinimalQuoteInvestment(s.Market, grid) - assert.InDelta(t, 60.46, minQuoteInvestment.Float64(), 0.01) + assert.InDelta(t, 48.36, minQuoteInvestment.Float64(), 0.01) err := s.checkMinimalQuoteInvestment(grid) assert.NoError(t, err) @@ -1207,12 +1283,11 @@ func TestStrategy_checkMinimalQuoteInvestment(t *testing.T) { s := newTestStrategy() // 10_000 * 0.001 = 10USDT // 20_000 * 0.001 = 20USDT - // hence we should have at least: 20USDT * 10 grids s.QuoteInvestment = number(10_000) s.GridNum = 10 grid := s.newGrid() minQuoteInvestment := calculateMinimalQuoteInvestment(s.Market, grid) - assert.InDelta(t, 129.9999, minQuoteInvestment.Float64(), 0.01) + assert.InDelta(t, 103.999, minQuoteInvestment.Float64(), 0.01) err := s.checkMinimalQuoteInvestment(grid) assert.NoError(t, err) @@ -1225,11 +1300,11 @@ func TestStrategy_checkMinimalQuoteInvestment(t *testing.T) { grid := s.newGrid() minQuoteInvestment := calculateMinimalQuoteInvestment(s.Market, grid) - assert.InDelta(t, 14979.995499, minQuoteInvestment.Float64(), 0.001) + assert.InDelta(t, 11983.996400, minQuoteInvestment.Float64(), 0.001) err := s.checkMinimalQuoteInvestment(grid) assert.Error(t, err) - assert.EqualError(t, err, "need at least 14979.995500 USDT for quote investment, 10000.000000 USDT given") + assert.EqualError(t, err, "need at least 11983.996400 USDT for quote investment, 10000.000000 USDT given") }) } From e614741a48c56b89c415e34feab69fd2ed19a9e7 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 3 Nov 2023 18:22:33 +0800 Subject: [PATCH 04/33] grid2: add another test case for 0 baseGridNum --- pkg/strategy/grid2/strategy_test.go | 60 ++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/pkg/strategy/grid2/strategy_test.go b/pkg/strategy/grid2/strategy_test.go index 0110782dd..70d6f97ef 100644 --- a/pkg/strategy/grid2/strategy_test.go +++ b/pkg/strategy/grid2/strategy_test.go @@ -204,7 +204,7 @@ func TestStrategy_generateGridOrders(t *testing.T) { }, orders) }) - t.Run("base and quote #2", func(t *testing.T) { + t.Run("base and quote with predefined base grid num", func(t *testing.T) { gridNum := int64(22) upperPrice := number(35500.000000) lowerPrice := number(34450.000000) @@ -263,6 +263,64 @@ func TestStrategy_generateGridOrders(t *testing.T) { }, orders) }) + t.Run("base and quote", func(t *testing.T) { + gridNum := int64(22) + upperPrice := number(35500.000000) + lowerPrice := number(34450.000000) + quoteInvestment := number(20.0) + baseInvestment := number(0.010700) + lastPrice := number(34522.930000) + baseGridNum := int(0) + + s := newTestStrategy() + s.GridNum = gridNum + s.BaseGridNum = baseGridNum + s.LowerPrice = lowerPrice + s.UpperPrice = upperPrice + s.grid = NewGrid(lowerPrice, upperPrice, fixedpoint.NewFromInt(s.GridNum), s.Market.TickSize) + s.grid.CalculateArithmeticPins() + assert.Equal(t, 22, len(s.grid.Pins)) + + quantity, err := s.calculateBaseQuoteInvestmentQuantity(quoteInvestment, baseInvestment, lastPrice, s.grid.Pins) + assert.NoError(t, err) + assert.Equal(t, "0.00029006", quantity.String()) + + s.QuantityOrAmount.Quantity = quantity + + orders, err := s.generateGridOrders(quoteInvestment, baseInvestment, lastPrice) + assert.NoError(t, err) + if !assert.Equal(t, 21, len(orders)) { + for _, o := range orders { + t.Logf("- %s %s", o.Price.String(), o.Side) + } + } + + assertPriceSide(t, []PriceSideAssert{ + {number(35500.0), types.SideTypeSell}, + {number(35450.0), types.SideTypeSell}, + {number(35400.0), types.SideTypeSell}, + {number(35350.0), types.SideTypeSell}, + {number(35300.0), types.SideTypeSell}, + {number(35250.0), types.SideTypeSell}, + {number(35200.0), types.SideTypeSell}, + {number(35150.0), types.SideTypeSell}, + {number(35100.0), types.SideTypeSell}, + {number(35050.0), types.SideTypeSell}, + {number(35000.0), types.SideTypeSell}, + {number(34950.0), types.SideTypeSell}, + {number(34900.0), types.SideTypeSell}, + {number(34850.0), types.SideTypeSell}, + {number(34800.0), types.SideTypeSell}, + {number(34750.0), types.SideTypeSell}, + {number(34700.0), types.SideTypeSell}, + {number(34650.0), types.SideTypeSell}, + {number(34600.0), types.SideTypeSell}, + {number(34550.0), types.SideTypeSell}, + // -- 34500 should be empty + {number(34450.0), types.SideTypeBuy}, + }, orders) + }) + t.Run("base and quote with pre-calculated baseGridNumber", func(t *testing.T) { s := newTestStrategy() s.grid = NewGrid(s.LowerPrice, s.UpperPrice, fixedpoint.NewFromInt(s.GridNum), s.Market.TickSize) From 358aef770fb71346933f6a75aa4bc31230b97376 Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 6 Nov 2023 17:13:16 +0800 Subject: [PATCH 05/33] FIX: fix skip syncing active order --- pkg/strategy/grid2/active_order_recover.go | 11 ++++++++--- pkg/strategy/grid2/recover.go | 20 ++++++++++++++++++-- pkg/strategy/grid2/recover_test.go | 4 ++-- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index cfdaeac80..91b52ed68 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -2,6 +2,7 @@ package grid2 import ( "context" + "strings" "time" "github.com/c9s/bbgo/pkg/bbgo" @@ -125,9 +126,13 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { // sleep 100ms to avoid DDOS time.Sleep(100 * time.Millisecond) - if err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID); err != nil { - opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) - errs = multierr.Append(errs, err) + if err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID, syncBefore); err != nil { + if strings.Contains(err.Error(), "skip syncing active order") { + opts.logger.Infof("[ActiveOrderRecover] skip syncing active order #%d, because the updated_at is in 3 min", activeOrder.OrderID) + } else { + opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) + errs = multierr.Append(errs, err) + } continue } } diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index 0634b72d4..4a6456884 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strconv" + "strings" "time" "github.com/c9s/bbgo/pkg/bbgo" @@ -13,6 +14,8 @@ import ( "github.com/pkg/errors" ) +var syncWindow = -3 * time.Minute + /* Background knowledge 1. active orderbook add orders only when receive new order event or call Add/Update method manually @@ -91,6 +94,8 @@ func (s *Strategy) recover(ctx context.Context) error { pins := s.getGrid().Pins + syncBefore := time.Now().Add(syncWindow) + activeOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, activeOrders) openOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, openOrders) @@ -135,7 +140,14 @@ func (s *Strategy) recover(ctx context.Context) error { // case 2 if openOrderID == 0 { - syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, activeOrder.GetOrder().OrderID) + order := activeOrder.GetOrder() + if err := syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, order.OrderID, syncBefore); err != nil { + if strings.Contains(err.Error(), "skip syncing active order") { + s.logger.Infof("[Recover] skip handle active order #%d, because the updated_at is in 3 min", order.OrderID) + } else { + s.logger.WithError(err).Errorf("[Recover] unable to query order #%d", order.OrderID) + } + } continue } @@ -250,7 +262,7 @@ func buildTwinOrderBook(pins []Pin, orders []types.Order) (*TwinOrderBook, error return book, nil } -func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64) error { +func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64, syncBefore time.Time) error { updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{ Symbol: activeOrderBook.Symbol, OrderID: strconv.FormatUint(orderID, 10), @@ -260,6 +272,10 @@ func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, return err } + if updatedOrder.UpdateTime.After(syncBefore) { + return fmt.Errorf("skip syncing active order, because its updated_at is after %s", syncBefore) + } + activeOrderBook.Update(*updatedOrder) return nil diff --git a/pkg/strategy/grid2/recover_test.go b/pkg/strategy/grid2/recover_test.go index bdfd191ee..332c84d62 100644 --- a/pkg/strategy/grid2/recover_test.go +++ b/pkg/strategy/grid2/recover_test.go @@ -114,7 +114,7 @@ func TestSyncActiveOrder(t *testing.T) { OrderID: strconv.FormatUint(order.OrderID, 10), }).Return(&updatedOrder, nil) - if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { + if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now())) { return } @@ -144,7 +144,7 @@ func TestSyncActiveOrder(t *testing.T) { OrderID: strconv.FormatUint(order.OrderID, 10), }).Return(&updatedOrder, nil) - if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { + if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now())) { return } From dcff850c64e8b4ca7b160b1997b691b3b70da41c Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 6 Nov 2023 18:52:01 +0800 Subject: [PATCH 06/33] FEATURE: add ttl for position/grid2.profit_stats persistence --- pkg/strategy/grid2/profit_stats.go | 11 +++++++++++ pkg/strategy/grid2/strategy.go | 5 +++++ pkg/types/position.go | 11 +++++++++++ 3 files changed, 27 insertions(+) diff --git a/pkg/strategy/grid2/profit_stats.go b/pkg/strategy/grid2/profit_stats.go index cd8367c23..c39602937 100644 --- a/pkg/strategy/grid2/profit_stats.go +++ b/pkg/strategy/grid2/profit_stats.go @@ -24,6 +24,9 @@ type GridProfitStats struct { Market types.Market `json:"market,omitempty"` Since *time.Time `json:"since,omitempty"` InitialOrderID uint64 `json:"initialOrderID"` + + // ttl is the ttl to keep in persistence + ttl time.Duration } func newGridProfitStats(market types.Market) *GridProfitStats { @@ -40,6 +43,14 @@ func newGridProfitStats(market types.Market) *GridProfitStats { } } +func (s *GridProfitStats) SetTTL(ttl time.Duration) { + s.ttl = ttl +} + +func (s *GridProfitStats) Expiration() time.Duration { + return s.ttl +} + func (s *GridProfitStats) AddTrade(trade types.Trade) { if s.TotalFee == nil { s.TotalFee = make(map[string]fixedpoint.Value) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 0c8248d93..bc361a72b 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -177,6 +177,7 @@ type Strategy struct { GridProfitStats *GridProfitStats `persistence:"grid_profit_stats"` Position *types.Position `persistence:"position"` + PersistenceTTL types.Duration `json:"persistenceTTL"` // ExchangeSession is an injection field ExchangeSession *bbgo.ExchangeSession @@ -1835,13 +1836,17 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.ProfitSpread = s.Market.TruncatePrice(s.ProfitSpread) } + s.logger.Infof("ttl: %s", s.PersistenceTTL.Duration()) + if s.GridProfitStats == nil { s.GridProfitStats = newGridProfitStats(s.Market) } + s.GridProfitStats.SetTTL(s.PersistenceTTL.Duration()) if s.Position == nil { s.Position = types.NewPositionFromMarket(s.Market) } + s.Position.SetTTL(s.PersistenceTTL.Duration()) // initialize and register prometheus metrics if s.PrometheusLabels != nil { diff --git a/pkg/types/position.go b/pkg/types/position.go index 983c1c551..4b649a48f 100644 --- a/pkg/types/position.go +++ b/pkg/types/position.go @@ -65,6 +65,17 @@ type Position struct { // Modify position callbacks modifyCallbacks []func(baseQty fixedpoint.Value, quoteQty fixedpoint.Value, price fixedpoint.Value) + + // ttl is the ttl to keep in persistence + ttl time.Duration +} + +func (s *Position) SetTTL(ttl time.Duration) { + s.ttl = ttl +} + +func (s *Position) Expiration() time.Duration { + return s.ttl } func (p *Position) CsvHeader() []string { From 82ac8f184f9bdbe63a8cd868253ef99eb794f2d0 Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 6 Nov 2023 22:17:29 +0800 Subject: [PATCH 07/33] pkg/exchange: to periodically fetch the fee rate --- pkg/exchange/bybit/market_info_poller.go | 137 ++++++++++++++ pkg/exchange/bybit/market_info_poller_test.go | 173 ++++++++++++++++++ pkg/exchange/bybit/stream.go | 67 +------ pkg/exchange/bybit/stream_test.go | 135 +------------- 4 files changed, 328 insertions(+), 184 deletions(-) create mode 100644 pkg/exchange/bybit/market_info_poller.go create mode 100644 pkg/exchange/bybit/market_info_poller_test.go diff --git a/pkg/exchange/bybit/market_info_poller.go b/pkg/exchange/bybit/market_info_poller.go new file mode 100644 index 000000000..dac0172f9 --- /dev/null +++ b/pkg/exchange/bybit/market_info_poller.go @@ -0,0 +1,137 @@ +package bybit + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + "github.com/c9s/bbgo/pkg/util" +) + +const ( + // To maintain aligned fee rates, it's important to update fees frequently. + feeRatePollingPeriod = time.Minute +) + +type symbolFeeDetail struct { + bybitapi.FeeRate + + BaseCoin string + QuoteCoin string +} + +// feeRatePoller pulls the specified market data from bbgo QueryMarkets. +type feeRatePoller struct { + mu sync.Mutex + once sync.Once + client MarketInfoProvider + + symbolFeeDetail map[string]symbolFeeDetail +} + +func newFeeRatePoller(marketInfoProvider MarketInfoProvider) *feeRatePoller { + return &feeRatePoller{ + client: marketInfoProvider, + symbolFeeDetail: map[string]symbolFeeDetail{}, + } +} + +func (p *feeRatePoller) Start(ctx context.Context) { + p.once.Do(func() { + p.startLoop(ctx) + }) +} + +func (p *feeRatePoller) startLoop(ctx context.Context) { + ticker := time.NewTicker(feeRatePollingPeriod) + defer ticker.Stop() + + // Make sure the first poll should succeed by retrying with a shorter period. + _ = util.Retry(ctx, util.InfiniteRetry, 30*time.Second, + func() error { return p.poll(ctx) }, + func(e error) { log.WithError(e).Warn("failed to update fee rate") }) + + for { + select { + case <-ctx.Done(): + if err := ctx.Err(); !errors.Is(err, context.Canceled) { + log.WithError(err).Error("context done with error") + } + + return + case <-ticker.C: + if err := p.poll(ctx); err != nil { + log.WithError(err).Warn("failed to update fee rate") + } + } + } +} + +func (p *feeRatePoller) poll(ctx context.Context) error { + symbolFeeRate, err := p.getAllFeeRates(ctx) + if err != nil { + return err + } + + p.mu.Lock() + p.symbolFeeDetail = symbolFeeRate + p.mu.Unlock() + + return nil +} + +func (p *feeRatePoller) Get(symbol string) (symbolFeeDetail, error) { + p.mu.Lock() + defer p.mu.Unlock() + + fee, ok := p.symbolFeeDetail[symbol] + if !ok { + return symbolFeeDetail{}, fmt.Errorf("%s fee rate not found", symbol) + } + return fee, nil +} + +func (e *feeRatePoller) getAllFeeRates(ctx context.Context) (map[string]symbolFeeDetail, error) { + feeRates, err := e.client.GetAllFeeRates(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call get fee rates: %w", err) + } + + symbolMap := map[string]symbolFeeDetail{} + for _, f := range feeRates.List { + if _, found := symbolMap[f.Symbol]; !found { + symbolMap[f.Symbol] = symbolFeeDetail{FeeRate: f} + } + } + + mkts, err := e.client.QueryMarkets(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get markets: %w", err) + } + + // update base coin, quote coin into symbolFeeDetail + for _, mkt := range mkts { + feeRate, found := symbolMap[mkt.Symbol] + if !found { + continue + } + + feeRate.BaseCoin = mkt.BaseCurrency + feeRate.QuoteCoin = mkt.QuoteCurrency + + symbolMap[mkt.Symbol] = feeRate + } + + // remove trading pairs that are not present in spot market. + for k, v := range symbolMap { + if len(v.BaseCoin) == 0 || len(v.QuoteCoin) == 0 { + log.Debugf("related market not found: %s, skipping the associated trade", k) + delete(symbolMap, k) + } + } + + return symbolMap, nil +} diff --git a/pkg/exchange/bybit/market_info_poller_test.go b/pkg/exchange/bybit/market_info_poller_test.go new file mode 100644 index 000000000..f2b58c466 --- /dev/null +++ b/pkg/exchange/bybit/market_info_poller_test.go @@ -0,0 +1,173 @@ +package bybit + +import ( + "context" + "fmt" + "testing" + + "github.com/golang/mock/gomock" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + "github.com/c9s/bbgo/pkg/exchange/bybit/mocks" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func TestFeeRatePoller_getAllFeeRates(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + unknownErr := errors.New("unknown err") + + t.Run("succeeds", func(t *testing.T) { + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) + s := &feeRatePoller{ + client: mockMarketProvider, + } + + ctx := context.Background() + feeRates := bybitapi.FeeRates{ + List: []bybitapi.FeeRate{ + { + Symbol: "BTCUSDT", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + { + Symbol: "ETHUSDT", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + { + Symbol: "OPTIONCOIN", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + }, + } + + mkts := types.MarketMap{ + "BTCUSDT": types.Market{ + Symbol: "BTCUSDT", + QuoteCurrency: "USDT", + BaseCurrency: "BTC", + }, + "ETHUSDT": types.Market{ + Symbol: "ETHUSDT", + QuoteCurrency: "USDT", + BaseCurrency: "ETH", + }, + } + + mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(feeRates, nil).Times(1) + mockMarketProvider.EXPECT().QueryMarkets(ctx).Return(mkts, nil).Times(1) + + expFeeRates := map[string]symbolFeeDetail{ + "BTCUSDT": { + FeeRate: feeRates.List[0], + BaseCoin: "BTC", + QuoteCoin: "USDT", + }, + "ETHUSDT": { + FeeRate: feeRates.List[1], + BaseCoin: "ETH", + QuoteCoin: "USDT", + }, + } + symbolFeeDetails, err := s.getAllFeeRates(ctx) + assert.NoError(t, err) + assert.Equal(t, expFeeRates, symbolFeeDetails) + }) + + t.Run("failed to query markets", func(t *testing.T) { + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) + s := &feeRatePoller{ + client: mockMarketProvider, + } + + ctx := context.Background() + feeRates := bybitapi.FeeRates{ + List: []bybitapi.FeeRate{ + { + Symbol: "BTCUSDT", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + { + Symbol: "ETHUSDT", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + { + Symbol: "OPTIONCOIN", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + }, + } + + mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(feeRates, nil).Times(1) + mockMarketProvider.EXPECT().QueryMarkets(ctx).Return(nil, unknownErr).Times(1) + + symbolFeeDetails, err := s.getAllFeeRates(ctx) + assert.Equal(t, fmt.Errorf("failed to get markets: %w", unknownErr), err) + assert.Equal(t, map[string]symbolFeeDetail(nil), symbolFeeDetails) + }) + + t.Run("failed to get fee rates", func(t *testing.T) { + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) + s := &feeRatePoller{ + client: mockMarketProvider, + } + + ctx := context.Background() + + mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(bybitapi.FeeRates{}, unknownErr).Times(1) + + symbolFeeDetails, err := s.getAllFeeRates(ctx) + assert.Equal(t, fmt.Errorf("failed to call get fee rates: %w", unknownErr), err) + assert.Equal(t, map[string]symbolFeeDetail(nil), symbolFeeDetails) + }) +} + +func Test_feeRatePoller_Get(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) + t.Run("succeeds", func(t *testing.T) { + symbol := "BTCUSDT" + expFeeDetail := symbolFeeDetail{ + FeeRate: bybitapi.FeeRate{ + Symbol: symbol, + TakerFeeRate: fixedpoint.NewFromFloat(0.1), + MakerFeeRate: fixedpoint.NewFromFloat(0.2), + }, + BaseCoin: "BTC", + QuoteCoin: "USDT", + } + + s := &feeRatePoller{ + client: mockMarketProvider, + symbolFeeDetail: map[string]symbolFeeDetail{ + symbol: expFeeDetail, + }, + } + + res, err := s.Get(symbol) + assert.NoError(t, err) + assert.Equal(t, expFeeDetail, res) + }) + t.Run("succeeds", func(t *testing.T) { + symbol := "BTCUSDT" + s := &feeRatePoller{ + client: mockMarketProvider, + symbolFeeDetail: map[string]symbolFeeDetail{}, + } + + _, err := s.Get(symbol) + assert.ErrorContains(t, err, symbol) + }) +} diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index eb4137ed3..2bb262657 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -47,8 +47,7 @@ type Stream struct { key, secret string streamDataProvider StreamDataProvider - // TODO: update the fee rate at 7:00 am UTC; rotation required. - symbolFeeDetails map[string]*symbolFeeDetail + feeRateProvider *feeRatePoller bookEventCallbacks []func(e BookEvent) marketTradeEventCallbacks []func(e []MarketTradeEvent) @@ -65,13 +64,17 @@ func NewStream(key, secret string, userDataProvider StreamDataProvider) *Stream key: key, secret: secret, streamDataProvider: userDataProvider, + feeRateProvider: newFeeRatePoller(userDataProvider), } stream.SetEndpointCreator(stream.createEndpoint) stream.SetParser(stream.parseWebSocketEvent) stream.SetDispatcher(stream.dispatchEvent) stream.SetHeartBeat(stream.ping) - stream.SetBeforeConnect(stream.getAllFeeRates) + stream.SetBeforeConnect(func(ctx context.Context) error { + go stream.feeRateProvider.Start(ctx) + return nil + }) stream.OnConnect(stream.handlerConnect) stream.OnAuth(stream.handleAuthEvent) @@ -403,13 +406,13 @@ func (s *Stream) handleKLineEvent(klineEvent KLineEvent) { func (s *Stream) handleTradeEvent(events []TradeEvent) { for _, event := range events { - feeRate, found := s.symbolFeeDetails[event.Symbol] - if !found { - log.Warnf("unexpected symbol found, fee rate not supported, symbol: %s", event.Symbol) + feeRate, err := s.feeRateProvider.Get(event.Symbol) + if err != nil { + log.Warnf("failed to get fee rate by symbol: %s", event.Symbol) continue } - gTrade, err := event.toGlobalTrade(*feeRate) + gTrade, err := event.toGlobalTrade(feeRate) if err != nil { log.WithError(err).Errorf("unable to convert: %+v", event) continue @@ -417,53 +420,3 @@ func (s *Stream) handleTradeEvent(events []TradeEvent) { s.StandardStream.EmitTradeUpdate(*gTrade) } } - -type symbolFeeDetail struct { - bybitapi.FeeRate - - BaseCoin string - QuoteCoin string -} - -// getAllFeeRates retrieves all fee rates from the Bybit API and then fetches markets to ensure the base coin and quote coin -// are correct. -func (e *Stream) getAllFeeRates(ctx context.Context) error { - feeRates, err := e.streamDataProvider.GetAllFeeRates(ctx) - if err != nil { - return fmt.Errorf("failed to call get fee rates: %w", err) - } - - symbolMap := map[string]*symbolFeeDetail{} - for _, f := range feeRates.List { - if _, found := symbolMap[f.Symbol]; !found { - symbolMap[f.Symbol] = &symbolFeeDetail{FeeRate: f} - } - } - - mkts, err := e.streamDataProvider.QueryMarkets(ctx) - if err != nil { - return fmt.Errorf("failed to get markets: %w", err) - } - - // update base coin, quote coin into symbolFeeDetail - for _, mkt := range mkts { - feeRate, found := symbolMap[mkt.Symbol] - if !found { - continue - } - - feeRate.BaseCoin = mkt.BaseCurrency - feeRate.QuoteCoin = mkt.QuoteCurrency - } - - // remove trading pairs that are not present in spot market. - for k, v := range symbolMap { - if len(v.BaseCoin) == 0 || len(v.QuoteCoin) == 0 { - log.Debugf("related market not found: %s, skipping the associated trade", k) - delete(symbolMap, k) - } - } - - e.symbolFeeDetails = symbolMap - return nil -} diff --git a/pkg/exchange/bybit/stream_test.go b/pkg/exchange/bybit/stream_test.go index e1fec2af8..8ed651022 100644 --- a/pkg/exchange/bybit/stream_test.go +++ b/pkg/exchange/bybit/stream_test.go @@ -9,11 +9,9 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" - "github.com/c9s/bbgo/pkg/exchange/bybit/mocks" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/testutil" "github.com/c9s/bbgo/pkg/types" @@ -36,7 +34,7 @@ func getTestClientOrSkip(t *testing.T) *Stream { } func TestStream(t *testing.T) { - t.Skip() + //t.Skip() s := getTestClientOrSkip(t) symbols := []string{ @@ -70,12 +68,12 @@ func TestStream(t *testing.T) { err := s.Connect(context.Background()) assert.NoError(t, err) - s.OnBookSnapshot(func(book types.SliceOrderBook) { - t.Log("got snapshot", book) - }) - s.OnBookUpdate(func(book types.SliceOrderBook) { - t.Log("got update", book) - }) + //s.OnBookSnapshot(func(book types.SliceOrderBook) { + // t.Log("got snapshot", book) + //}) + //s.OnBookUpdate(func(book types.SliceOrderBook) { + // t.Log("got update", book) + //}) c := make(chan struct{}) <-c }) @@ -175,7 +173,7 @@ func TestStream(t *testing.T) { assert.NoError(t, err) s.OnTradeUpdate(func(trade types.Trade) { - t.Log("got update", trade) + t.Log("got update", trade.Fee, trade.FeeCurrency, trade) }) c := make(chan struct{}) <-c @@ -467,120 +465,3 @@ func Test_convertSubscription(t *testing.T) { assert.Equal(t, genTopic(TopicTypeMarketTrade, "BTCUSDT"), res) }) } - -func TestStream_getFeeRate(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - unknownErr := errors.New("unknown err") - - t.Run("succeeds", func(t *testing.T) { - mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) - s := &Stream{ - streamDataProvider: mockMarketProvider, - } - - ctx := context.Background() - feeRates := bybitapi.FeeRates{ - List: []bybitapi.FeeRate{ - { - Symbol: "BTCUSDT", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - { - Symbol: "ETHUSDT", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - { - Symbol: "OPTIONCOIN", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - }, - } - - mkts := types.MarketMap{ - "BTCUSDT": types.Market{ - Symbol: "BTCUSDT", - QuoteCurrency: "USDT", - BaseCurrency: "BTC", - }, - "ETHUSDT": types.Market{ - Symbol: "ETHUSDT", - QuoteCurrency: "USDT", - BaseCurrency: "ETH", - }, - } - - mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(feeRates, nil).Times(1) - mockMarketProvider.EXPECT().QueryMarkets(ctx).Return(mkts, nil).Times(1) - - expFeeRates := map[string]*symbolFeeDetail{ - "BTCUSDT": { - FeeRate: feeRates.List[0], - BaseCoin: "BTC", - QuoteCoin: "USDT", - }, - "ETHUSDT": { - FeeRate: feeRates.List[1], - BaseCoin: "ETH", - QuoteCoin: "USDT", - }, - } - err := s.getAllFeeRates(ctx) - assert.NoError(t, err) - assert.Equal(t, expFeeRates, s.symbolFeeDetails) - }) - - t.Run("failed to query markets", func(t *testing.T) { - mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) - s := &Stream{ - streamDataProvider: mockMarketProvider, - } - - ctx := context.Background() - feeRates := bybitapi.FeeRates{ - List: []bybitapi.FeeRate{ - { - Symbol: "BTCUSDT", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - { - Symbol: "ETHUSDT", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - { - Symbol: "OPTIONCOIN", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - }, - } - - mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(feeRates, nil).Times(1) - mockMarketProvider.EXPECT().QueryMarkets(ctx).Return(nil, unknownErr).Times(1) - - err := s.getAllFeeRates(ctx) - assert.Equal(t, fmt.Errorf("failed to get markets: %w", unknownErr), err) - assert.Equal(t, map[string]*symbolFeeDetail(nil), s.symbolFeeDetails) - }) - - t.Run("failed to get fee rates", func(t *testing.T) { - mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) - s := &Stream{ - streamDataProvider: mockMarketProvider, - } - - ctx := context.Background() - - mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(bybitapi.FeeRates{}, unknownErr).Times(1) - - err := s.getAllFeeRates(ctx) - assert.Equal(t, fmt.Errorf("failed to call get fee rates: %w", unknownErr), err) - assert.Equal(t, map[string]*symbolFeeDetail(nil), s.symbolFeeDetails) - }) -} From f595cc9cc05bac4501377af6617a9b5dd61175cd Mon Sep 17 00:00:00 2001 From: Edwin Date: Sun, 5 Nov 2023 23:41:57 +0800 Subject: [PATCH 08/33] pkg/exchange: add query open orders --- pkg/exchange/bitget/bitgetapi/v2/client.go | 17 ++ .../bitget/bitgetapi/v2/client_test.go | 42 ++++ .../v2/get_unfilled_orders_request.go | 50 ++++ .../get_unfilled_orders_request_requestgen.go | 221 ++++++++++++++++++ pkg/exchange/bitget/bitgetapi/v2/types.go | 42 ++++ pkg/exchange/bitget/convert.go | 99 ++++++++ pkg/exchange/bitget/convert_test.go | 129 ++++++++++ pkg/exchange/bitget/exchange.go | 52 ++++- 8 files changed, 647 insertions(+), 5 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/client.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/client_test.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/types.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client.go b/pkg/exchange/bitget/bitgetapi/v2/client.go new file mode 100644 index 000000000..3a2b2204d --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/client.go @@ -0,0 +1,17 @@ +package bitgetapi + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" +) + +type APIResponse = bitgetapi.APIResponse + +type Client struct { + Client requestgen.AuthenticatedAPIClient +} + +func NewClient(client *bitgetapi.RestClient) *Client { + return &Client{Client: client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go new file mode 100644 index 000000000..21c86dfc6 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -0,0 +1,42 @@ +package bitgetapi + +import ( + "context" + "os" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "github.com/c9s/bbgo/pkg/testutil" +) + +func getTestClientOrSkip(t *testing.T) *Client { + if b, _ := strconv.ParseBool(os.Getenv("CI")); b { + t.Skip("skip test for CI") + } + + key, secret, ok := testutil.IntegrationTestConfigured(t, "BITGET") + if !ok { + t.Skip("BITGET_* env vars are not configured") + return nil + } + + client := bitgetapi.NewClient() + client.Auth(key, secret, os.Getenv("BITGET_API_PASSPHRASE")) + + return NewClient(client) +} + +func TestClient(t *testing.T) { + client := getTestClientOrSkip(t) + ctx := context.Background() + + t.Run("GetUnfilledOrdersRequest", func(t *testing.T) { + req := client.NewGetUnfilledOrdersRequest().StartTime(1) + resp, err := req.Do(ctx) + assert.NoError(t, err) + t.Logf("resp: %+v", resp) + }) +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go new file mode 100644 index 000000000..178b31bba --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go @@ -0,0 +1,50 @@ +package bitgetapi + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +type UnfilledOrder struct { + UserId types.StrInt64 `json:"userId"` + Symbol string `json:"symbol"` + // OrderId are always numeric. It's confirmed with official customer service. https://t.me/bitgetOpenapi/24172 + OrderId types.StrInt64 `json:"orderId"` + ClientOrderId string `json:"clientOid"` + PriceAvg fixedpoint.Value `json:"priceAvg"` + // Size is base coin when orderType=limit; quote coin when orderType=market + Size fixedpoint.Value `json:"size"` + OrderType OrderType `json:"orderType"` + Side SideType `json:"side"` + Status OrderStatus `json:"status"` + BasePrice fixedpoint.Value `json:"basePrice"` + BaseVolume fixedpoint.Value `json:"baseVolume"` + QuoteVolume fixedpoint.Value `json:"quoteVolume"` + EnterPointSource string `json:"enterPointSource"` + OrderSource string `json:"orderSource"` + CTime types.MillisecondTimestamp `json:"cTime"` + UTime types.MillisecondTimestamp `json:"uTime"` +} + +//go:generate GetRequest -url "/api/v2/spot/trade/unfilled-orders" -type GetUnfilledOrdersRequest -responseDataType []UnfilledOrder +type GetUnfilledOrdersRequest struct { + client requestgen.AuthenticatedAPIClient + + symbol *string `param:"symbol,query"` + // Limit number default 100 max 100 + limit *string `param:"limit,query"` + // idLessThan requests the content on the page before this ID (older data), the value input should be the orderId of the corresponding interface. + idLessThan *string `param:"idLessThan,query"` + startTime *int64 `param:"startTime,query"` + endTime *int64 `param:"endTime,query"` + orderId *string `param:"orderId,query"` +} + +func (c *Client) NewGetUnfilledOrdersRequest() *GetUnfilledOrdersRequest { + return &GetUnfilledOrdersRequest{client: c.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go new file mode 100644 index 000000000..a3bb59819 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go @@ -0,0 +1,221 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/trade/unfilled-orders -type GetUnfilledOrdersRequest -responseDataType []UnfilledOrder"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (g *GetUnfilledOrdersRequest) Symbol(symbol string) *GetUnfilledOrdersRequest { + g.symbol = &symbol + return g +} + +func (g *GetUnfilledOrdersRequest) Limit(limit string) *GetUnfilledOrdersRequest { + g.limit = &limit + return g +} + +func (g *GetUnfilledOrdersRequest) IdLessThan(idLessThan string) *GetUnfilledOrdersRequest { + g.idLessThan = &idLessThan + return g +} + +func (g *GetUnfilledOrdersRequest) StartTime(startTime int64) *GetUnfilledOrdersRequest { + g.startTime = &startTime + return g +} + +func (g *GetUnfilledOrdersRequest) EndTime(endTime int64) *GetUnfilledOrdersRequest { + g.endTime = &endTime + return g +} + +func (g *GetUnfilledOrdersRequest) OrderId(orderId string) *GetUnfilledOrdersRequest { + g.orderId = &orderId + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetUnfilledOrdersRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + if g.symbol != nil { + symbol := *g.symbol + + // assign parameter of symbol + params["symbol"] = symbol + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + // check idLessThan field -> json key idLessThan + if g.idLessThan != nil { + idLessThan := *g.idLessThan + + // assign parameter of idLessThan + params["idLessThan"] = idLessThan + } else { + } + // check startTime field -> json key startTime + if g.startTime != nil { + startTime := *g.startTime + + // assign parameter of startTime + params["startTime"] = startTime + } else { + } + // check endTime field -> json key endTime + if g.endTime != nil { + endTime := *g.endTime + + // assign parameter of endTime + params["endTime"] = endTime + } else { + } + // check orderId field -> json key orderId + if g.orderId != nil { + orderId := *g.orderId + + // assign parameter of orderId + params["orderId"] = orderId + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetUnfilledOrdersRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetUnfilledOrdersRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetUnfilledOrdersRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetUnfilledOrdersRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetUnfilledOrdersRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetUnfilledOrdersRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetUnfilledOrdersRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetUnfilledOrdersRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (g *GetUnfilledOrdersRequest) Do(ctx context.Context) ([]UnfilledOrder, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + apiURL := "/api/v2/spot/trade/unfilled-orders" + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + var data []UnfilledOrder + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/types.go b/pkg/exchange/bitget/bitgetapi/v2/types.go new file mode 100644 index 000000000..82a91859c --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/types.go @@ -0,0 +1,42 @@ +package bitgetapi + +type SideType string + +const ( + SideTypeBuy SideType = "buy" + SideTypeSell SideType = "sell" +) + +type OrderType string + +const ( + OrderTypeLimit OrderType = "limit" + OrderTypeMarket OrderType = "market" +) + +type OrderForce string + +const ( + OrderForceGTC OrderForce = "gtc" + OrderForcePostOnly OrderForce = "post_only" + OrderForceFOK OrderForce = "fok" + OrderForceIOC OrderForce = "ioc" +) + +type OrderStatus string + +const ( + OrderStatusInit OrderStatus = "init" + OrderStatusNew OrderStatus = "new" + OrderStatusLive OrderStatus = "live" + OrderStatusPartialFilled OrderStatus = "partially_filled" + OrderStatusFilled OrderStatus = "filled" + OrderStatusCancelled OrderStatus = "cancelled" +) + +func (o OrderStatus) IsWorking() bool { + return o == OrderStatusInit || + o == OrderStatusNew || + o == OrderStatusLive || + o == OrderStatusPartialFilled +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 1339089fd..a4a9756d0 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -1,10 +1,13 @@ package bitget import ( + "fmt" "math" + "strconv" "strings" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -59,3 +62,99 @@ func toGlobalTicker(ticker bitgetapi.Ticker) types.Ticker { Sell: ticker.SellOne, } } + +func toGlobalSideType(side v2.SideType) (types.SideType, error) { + switch side { + case v2.SideTypeBuy: + return types.SideTypeBuy, nil + + case v2.SideTypeSell: + return types.SideTypeSell, nil + + default: + return types.SideType(side), fmt.Errorf("unexpected side: %s", side) + } +} + +func toGlobalOrderType(s v2.OrderType) (types.OrderType, error) { + switch s { + case v2.OrderTypeMarket: + return types.OrderTypeMarket, nil + + case v2.OrderTypeLimit: + return types.OrderTypeLimit, nil + + default: + return types.OrderType(s), fmt.Errorf("unexpected order type: %s", s) + } +} + +func toGlobalOrderStatus(status v2.OrderStatus) (types.OrderStatus, error) { + switch status { + case v2.OrderStatusInit, v2.OrderStatusNew, v2.OrderStatusLive: + return types.OrderStatusNew, nil + + case v2.OrderStatusPartialFilled: + return types.OrderStatusPartiallyFilled, nil + + case v2.OrderStatusFilled: + return types.OrderStatusFilled, nil + + case v2.OrderStatusCancelled: + return types.OrderStatusCanceled, nil + + default: + return types.OrderStatus(status), fmt.Errorf("unexpected order status: %s", status) + } +} + +// unfilledOrderToGlobalOrder convert the local order to global. +// +// Note that the quantity unit, according official document: Base coin when orderType=limit; Quote coin when orderType=market +// https://bitgetlimited.github.io/apidoc/zh/spot/#19671a1099 +func unfilledOrderToGlobalOrder(order v2.UnfilledOrder) (*types.Order, error) { + side, err := toGlobalSideType(order.Side) + if err != nil { + return nil, err + } + + orderType, err := toGlobalOrderType(order.OrderType) + if err != nil { + return nil, err + } + + status, err := toGlobalOrderStatus(order.Status) + if err != nil { + return nil, err + } + + qty := order.Size + price := order.PriceAvg + + // The market order will be executed immediately, so this check is used to handle corner cases. + if orderType == types.OrderTypeMarket { + qty = order.BaseVolume + log.Warnf("!!! The price(%f) and quantity(%f) are not verified for market orders, because we only receive limit orders in the test environment !!!", price.Float64(), qty.Float64()) + } + + return &types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: order.ClientOrderId, + Symbol: order.Symbol, + Side: side, + Type: orderType, + Quantity: qty, + Price: price, + // Bitget does not include the "time-in-force" field in its API response for spot trading, so we set GTC. + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(order.OrderId), + UUID: strconv.FormatInt(int64(order.OrderId), 10), + Status: status, + ExecutedQuantity: order.BaseVolume, + IsWorking: order.Status.IsWorking(), + CreationTime: types.Time(order.CTime.Time()), + UpdateTime: types.Time(order.UTime.Time()), + }, nil +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 770e4e5b8..8b8bc61c7 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -1,11 +1,13 @@ package bitget import ( + "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -143,3 +145,130 @@ func Test_toGlobalTicker(t *testing.T) { Sell: fixedpoint.NewFromFloat(24014.06), }, toGlobalTicker(ticker)) } + +func Test_toGlobalSideType(t *testing.T) { + side, err := toGlobalSideType(v2.SideTypeBuy) + assert.NoError(t, err) + assert.Equal(t, types.SideTypeBuy, side) + + side, err = toGlobalSideType(v2.SideTypeSell) + assert.NoError(t, err) + assert.Equal(t, types.SideTypeSell, side) + + _, err = toGlobalSideType("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_toGlobalOrderType(t *testing.T) { + orderType, err := toGlobalOrderType(v2.OrderTypeMarket) + assert.NoError(t, err) + assert.Equal(t, types.OrderTypeMarket, orderType) + + orderType, err = toGlobalOrderType(v2.OrderTypeLimit) + assert.NoError(t, err) + assert.Equal(t, types.OrderTypeLimit, orderType) + + _, err = toGlobalOrderType("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_toGlobalOrderStatus(t *testing.T) { + status, err := toGlobalOrderStatus(v2.OrderStatusInit) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusNew, status) + + status, err = toGlobalOrderStatus(v2.OrderStatusNew) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusNew, status) + + status, err = toGlobalOrderStatus(v2.OrderStatusLive) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusNew, status) + + status, err = toGlobalOrderStatus(v2.OrderStatusFilled) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusFilled, status) + + status, err = toGlobalOrderStatus(v2.OrderStatusPartialFilled) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusPartiallyFilled, status) + + status, err = toGlobalOrderStatus(v2.OrderStatusCancelled) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusCanceled, status) + + _, err = toGlobalOrderStatus("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_unfilledOrderToGlobalOrder(t *testing.T) { + var ( + assert = assert.New(t) + orderId = 1105087175647989764 + unfilledOrder = v2.UnfilledOrder{ + Symbol: "BTCUSDT", + OrderId: types.StrInt64(orderId), + ClientOrderId: "74b86af3-6098-479c-acac-bfb074c067f3", + PriceAvg: fixedpoint.NewFromFloat(1.2), + Size: fixedpoint.NewFromFloat(5), + OrderType: v2.OrderTypeLimit, + Side: v2.SideTypeBuy, + Status: v2.OrderStatusLive, + BasePrice: fixedpoint.NewFromFloat(0), + BaseVolume: fixedpoint.NewFromFloat(0), + QuoteVolume: fixedpoint.NewFromFloat(0), + EnterPointSource: "API", + OrderSource: "normal", + CTime: types.NewMillisecondTimestampFromInt(1660704288118), + UTime: types.NewMillisecondTimestampFromInt(1660704288118), + } + ) + + t.Run("succeeds", func(t *testing.T) { + order, err := unfilledOrderToGlobalOrder(unfilledOrder) + assert.NoError(err) + assert.Equal(&types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: "74b86af3-6098-479c-acac-bfb074c067f3", + Symbol: "BTCUSDT", + Side: types.SideTypeBuy, + Type: types.OrderTypeLimit, + Quantity: fixedpoint.NewFromFloat(5), + Price: fixedpoint.NewFromFloat(1.2), + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(orderId), + UUID: strconv.FormatInt(int64(orderId), 10), + Status: types.OrderStatusNew, + ExecutedQuantity: fixedpoint.NewFromFloat(0), + IsWorking: true, + CreationTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + UpdateTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + }, order) + }) + + t.Run("failed to convert side", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.Side = "xxx" + + _, err := unfilledOrderToGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) + + t.Run("failed to convert oder type", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.OrderType = "xxx" + + _, err := unfilledOrderToGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) + + t.Run("failed to convert oder status", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.Status = "xxx" + + _, err := unfilledOrderToGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 700bd2ea7..e4eef08a7 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -3,18 +3,24 @@ package bitget import ( "context" "fmt" + "strconv" "time" "github.com/sirupsen/logrus" "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/types" ) -const ID = "bitget" +const ( + ID = "bitget" -const PlatformToken = "BGB" + PlatformToken = "BGB" + + queryOpenOrdersLimit = 100 +) var log = logrus.WithFields(logrus.Fields{ "exchange": ID, @@ -29,12 +35,15 @@ var ( queryTickerRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) // queryTickersRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-all-tickers queryTickersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // queryOpenOrdersRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Get-Unfilled-Orders + queryOpenOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) ) type Exchange struct { key, secret, passphrase string - client *bitgetapi.RestClient + client *bitgetapi.RestClient + v2Client *v2.Client } func New(key, secret, passphrase string) *Exchange { @@ -49,6 +58,7 @@ func New(key, secret, passphrase string) *Exchange { secret: secret, passphrase: passphrase, client: client, + v2Client: v2.NewClient(client), } } @@ -174,8 +184,40 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr } func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders []types.Order, err error) { - // TODO implement me - panic("implement me") + var nextCursor types.StrInt64 + for { + if err := queryOpenOrdersRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("open order rate limiter wait error: %w", err) + } + + req := e.v2Client.NewGetUnfilledOrdersRequest(). + Symbol(symbol). + Limit(strconv.FormatInt(queryOpenOrdersLimit, 10)) + if nextCursor != 0 { + req.IdLessThan(strconv.FormatInt(int64(nextCursor), 10)) + } + + openOrders, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query open orders: %w", err) + } + + for _, o := range openOrders { + order, err := unfilledOrderToGlobalOrder(o) + if err != nil { + return nil, fmt.Errorf("failed to convert order, err: %v", err) + } + + orders = append(orders, *order) + } + + if len(openOrders) != queryOpenOrdersLimit { + break + } + nextCursor = openOrders[len(openOrders)-1].OrderId + } + + return orders, nil } func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) error { From 5b641957974b122c4215090a8e9e0e9f07d7e97a Mon Sep 17 00:00:00 2001 From: Sven Woldt Date: Tue, 7 Nov 2023 00:41:59 +0100 Subject: [PATCH 09/33] update createSymbolReport --- pkg/cmd/backtest.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 44edaa5c7..c8884cb7d 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -531,10 +531,7 @@ var BacktestCmd = &cobra.Command{ continue } - tradeState := sessionTradeStats[session.Name][symbol] - profitFactor := tradeState.ProfitFactor - winningRatio := tradeState.WinningRatio - intervalProfits := tradeState.IntervalProfits[types.Interval1d] + tradeStats := sessionTradeStats[session.Name][symbol] symbolReport, err := createSymbolReport(userConfig, session, symbol, trades.Copy(), tradeStats) if err != nil { return err @@ -546,8 +543,8 @@ var BacktestCmd = &cobra.Command{ summaryReport.TotalUnrealizedProfit = symbolReport.PnL.UnrealizedProfit summaryReport.InitialEquityValue = summaryReport.InitialEquityValue.Add(symbolReport.InitialEquityValue()) summaryReport.FinalEquityValue = summaryReport.FinalEquityValue.Add(symbolReport.FinalEquityValue()) - summaryReport.TotalGrossProfit.Add(symbolReport.PnL.GrossProfit) - summaryReport.TotalGrossLoss.Add(symbolReport.PnL.GrossLoss) + summaryReport.TotalGrossProfit = summaryReport.TotalGrossProfit.Add(symbolReport.PnL.GrossProfit) + summaryReport.TotalGrossLoss = summaryReport.TotalGrossLoss.Add(symbolReport.PnL.GrossLoss) // write report to a file if generatingReport { From c8becbe4f5f558afbcdbaa9e7896bdcf9e051ab7 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 7 Nov 2023 10:56:19 +0800 Subject: [PATCH 10/33] bbgo.sync when syncActiveOrders --- pkg/strategy/grid2/active_order_recover.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index cfdaeac80..55b48fcdf 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -30,7 +30,7 @@ func (s *Strategy) initializeRecoverC() bool { if s.recoverC == nil { s.logger.Info("initializing recover channel") - s.recoverC = make(chan struct{}, 1) + s.recoverC = make(chan struct{}, 10) } else { s.logger.Info("recover channel is already initialized, trigger active orders recover") isInitialize = true @@ -66,22 +66,26 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { exchange: s.session.Exchange, } + var lastRecoverTime time.Time + for { select { - case <-ctx.Done(): return case <-ticker.C: - if err := syncActiveOrders(ctx, opts); err != nil { - log.WithError(err).Errorf("unable to sync active orders") - } - + s.recoverC <- struct{}{} + bbgo.Sync(ctx, s) case <-s.recoverC: - if err := syncActiveOrders(ctx, opts); err != nil { - log.WithError(err).Errorf("unable to sync active orders") + if !time.Now().After(lastRecoverTime.Add(10 * time.Minute)) { + continue } + if err := syncActiveOrders(ctx, opts); err != nil { + log.WithError(err).Errorf("unable to sync active orders") + } else { + lastRecoverTime = time.Now() + } } } } From 7de49155eb515e5c0a56e1896a88f78ce0a39d30 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 7 Nov 2023 13:30:58 +0800 Subject: [PATCH 11/33] fix --- pkg/strategy/grid2/profit_stats.go | 3 +++ pkg/strategy/grid2/strategy.go | 2 +- pkg/types/position.go | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/strategy/grid2/profit_stats.go b/pkg/strategy/grid2/profit_stats.go index c39602937..a770b67fe 100644 --- a/pkg/strategy/grid2/profit_stats.go +++ b/pkg/strategy/grid2/profit_stats.go @@ -44,6 +44,9 @@ func newGridProfitStats(market types.Market) *GridProfitStats { } func (s *GridProfitStats) SetTTL(ttl time.Duration) { + if ttl.Nanoseconds() <= 0 { + return + } s.ttl = ttl } diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index bc361a72b..620d91ce1 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1836,7 +1836,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.ProfitSpread = s.Market.TruncatePrice(s.ProfitSpread) } - s.logger.Infof("ttl: %s", s.PersistenceTTL.Duration()) + s.logger.Infof("persistence ttl: %s", s.PersistenceTTL.Duration()) if s.GridProfitStats == nil { s.GridProfitStats = newGridProfitStats(s.Market) diff --git a/pkg/types/position.go b/pkg/types/position.go index 4b649a48f..589fa2a2b 100644 --- a/pkg/types/position.go +++ b/pkg/types/position.go @@ -71,6 +71,9 @@ type Position struct { } func (s *Position) SetTTL(ttl time.Duration) { + if ttl.Nanoseconds() <= 0 { + return + } s.ttl = ttl } From df2fd170db8a5a1298ef719ad5f8a98db3a1906c Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 7 Nov 2023 14:39:29 +0800 Subject: [PATCH 12/33] return bool to let syncActiveOrderBook really sync or skip --- pkg/strategy/grid2/active_order_recover.go | 21 +++++------------ pkg/strategy/grid2/recover.go | 26 ++++++++++------------ pkg/strategy/grid2/recover_test.go | 6 +++-- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 91b52ed68..16b93e9fd 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -2,7 +2,6 @@ package grid2 import ( "context" - "strings" "time" "github.com/c9s/bbgo/pkg/bbgo" @@ -118,22 +117,12 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { delete(openOrdersMap, activeOrder.OrderID) } else { opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) - if activeOrder.UpdateTime.After(syncBefore) { - opts.logger.Infof("active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID) - continue - } - // sleep 100ms to avoid DDOS - time.Sleep(100 * time.Millisecond) - - if err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID, syncBefore); err != nil { - if strings.Contains(err.Error(), "skip syncing active order") { - opts.logger.Infof("[ActiveOrderRecover] skip syncing active order #%d, because the updated_at is in 3 min", activeOrder.OrderID) - } else { - opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) - errs = multierr.Append(errs, err) - } - continue + if isActiveOrderBookUpdated, err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID, syncBefore); err != nil { + opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) + errs = multierr.Append(errs, err) + } else if !isActiveOrderBookUpdated { + opts.logger.Infof("[ActiveOrderRecover] active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID) } } } diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index 4a6456884..7859a84af 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strconv" - "strings" "time" "github.com/c9s/bbgo/pkg/bbgo" @@ -141,12 +140,10 @@ func (s *Strategy) recover(ctx context.Context) error { // case 2 if openOrderID == 0 { order := activeOrder.GetOrder() - if err := syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, order.OrderID, syncBefore); err != nil { - if strings.Contains(err.Error(), "skip syncing active order") { - s.logger.Infof("[Recover] skip handle active order #%d, because the updated_at is in 3 min", order.OrderID) - } else { - s.logger.WithError(err).Errorf("[Recover] unable to query order #%d", order.OrderID) - } + if isActiveOrderBookUpdated, err := syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, order.OrderID, syncBefore); err != nil { + s.logger.WithError(err).Errorf("[Recover] unable to query order #%d", order.OrderID) + } else if !isActiveOrderBookUpdated { + s.logger.Infof("[Recover] active order #%d is updated in 3 min, skip updating...", order.OrderID) } continue } @@ -262,23 +259,24 @@ func buildTwinOrderBook(pins []Pin, orders []types.Order) (*TwinOrderBook, error return book, nil } -func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64, syncBefore time.Time) error { +func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64, syncBefore time.Time) (bool, error) { updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{ Symbol: activeOrderBook.Symbol, OrderID: strconv.FormatUint(orderID, 10), }) + isActiveOrderBookUpdated := false + if err != nil { - return err + return isActiveOrderBookUpdated, err } - if updatedOrder.UpdateTime.After(syncBefore) { - return fmt.Errorf("skip syncing active order, because its updated_at is after %s", syncBefore) + isActiveOrderBookUpdated = updatedOrder.UpdateTime.Before(syncBefore) + if isActiveOrderBookUpdated { + activeOrderBook.Update(*updatedOrder) } - activeOrderBook.Update(*updatedOrder) - - return nil + return isActiveOrderBookUpdated, nil } func queryTradesToUpdateTwinOrderBook( diff --git a/pkg/strategy/grid2/recover_test.go b/pkg/strategy/grid2/recover_test.go index 332c84d62..fbd6ecb7c 100644 --- a/pkg/strategy/grid2/recover_test.go +++ b/pkg/strategy/grid2/recover_test.go @@ -114,7 +114,8 @@ func TestSyncActiveOrder(t *testing.T) { OrderID: strconv.FormatUint(order.OrderID, 10), }).Return(&updatedOrder, nil) - if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now())) { + _, err := syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now()) + if !assert.NoError(err) { return } @@ -144,7 +145,8 @@ func TestSyncActiveOrder(t *testing.T) { OrderID: strconv.FormatUint(order.OrderID, 10), }).Return(&updatedOrder, nil) - if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now())) { + _, err := syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now()) + if !assert.NoError(err) { return } From 2049e71cf6bfd382c7dc3118a91bd568d6463e09 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 7 Nov 2023 14:53:00 +0800 Subject: [PATCH 13/33] pkg/exchange: rm the retry --- pkg/exchange/bybit/market_info_poller.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/exchange/bybit/market_info_poller.go b/pkg/exchange/bybit/market_info_poller.go index dac0172f9..6266c8d29 100644 --- a/pkg/exchange/bybit/market_info_poller.go +++ b/pkg/exchange/bybit/market_info_poller.go @@ -46,14 +46,13 @@ func (p *feeRatePoller) Start(ctx context.Context) { } func (p *feeRatePoller) startLoop(ctx context.Context) { + err := p.poll(ctx) + if err != nil { + log.WithError(err).Warn("failed to initialize the fee rate, the ticker is scheduled to update it subsequently") + } + ticker := time.NewTicker(feeRatePollingPeriod) defer ticker.Stop() - - // Make sure the first poll should succeed by retrying with a shorter period. - _ = util.Retry(ctx, util.InfiniteRetry, 30*time.Second, - func() error { return p.poll(ctx) }, - func(e error) { log.WithError(e).Warn("failed to update fee rate") }) - for { select { case <-ctx.Done(): From e6fc0067477d61a56da611e684547f2a2242e013 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 7 Nov 2023 15:21:48 +0800 Subject: [PATCH 14/33] recoverC back to size 1 --- pkg/strategy/grid2/active_order_recover.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 55b48fcdf..5c64488b4 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -30,7 +30,7 @@ func (s *Strategy) initializeRecoverC() bool { if s.recoverC == nil { s.logger.Info("initializing recover channel") - s.recoverC = make(chan struct{}, 10) + s.recoverC = make(chan struct{}, 1) } else { s.logger.Info("recover channel is already initialized, trigger active orders recover") isInitialize = true From 4a40c8bea28cc63a1c70cdddf9a68dc495e2c1bd Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 7 Nov 2023 17:00:29 +0800 Subject: [PATCH 15/33] refactor --- pkg/strategy/grid2/active_order_recover.go | 10 +++++++--- pkg/strategy/grid2/recover.go | 18 +++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 16b93e9fd..87db11bf9 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -116,12 +116,16 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { // no need to sync active order already in active orderbook, because we only need to know if it filled or not. delete(openOrdersMap, activeOrder.OrderID) } else { - opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) + opts.logger.Infof("[ActiveOrderRecover] found active order #%d is not in the open orders, updating...", activeOrder.OrderID) - if isActiveOrderBookUpdated, err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID, syncBefore); err != nil { + isActiveOrderBookUpdated, err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID, syncBefore) + if err != nil { opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) errs = multierr.Append(errs, err) - } else if !isActiveOrderBookUpdated { + continue + } + + if !isActiveOrderBookUpdated { opts.logger.Infof("[ActiveOrderRecover] active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID) } } diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index 7859a84af..9418dd5a8 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -98,8 +98,8 @@ func (s *Strategy) recover(ctx context.Context) error { activeOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, activeOrders) openOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, openOrders) - s.logger.Infof("active orders' twin orderbook\n%s", activeOrdersInTwinOrderBook.String()) - s.logger.Infof("open orders in twin orderbook\n%s", openOrdersInTwinOrderBook.String()) + s.logger.Infof("[Recover] active orders' twin orderbook\n%s", activeOrdersInTwinOrderBook.String()) + s.logger.Infof("[Recover] open orders in twin orderbook\n%s", openOrdersInTwinOrderBook.String()) // remove index 0, because twin orderbook's price is from the second one pins = pins[1:] @@ -131,7 +131,9 @@ func (s *Strategy) recover(ctx context.Context) error { // case 1 if activeOrderID == 0 { - activeOrderBook.Add(openOrder.GetOrder()) + order := openOrder.GetOrder() + s.logger.Infof("[Recover] found open order #%d is not in the active orderbook, adding...", order.OrderID) + activeOrderBook.Add(order) // also add open orders into active order's twin orderbook, we will use this active orderbook to recover empty price grid activeOrdersInTwinOrderBook.AddTwinOrder(v, openOrder) continue @@ -140,11 +142,17 @@ func (s *Strategy) recover(ctx context.Context) error { // case 2 if openOrderID == 0 { order := activeOrder.GetOrder() - if isActiveOrderBookUpdated, err := syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, order.OrderID, syncBefore); err != nil { + s.logger.Infof("[Recover] found active order #%d is not in the open orders, updating...", order.OrderID) + isActiveOrderBookUpdated, err := syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, order.OrderID, syncBefore) + if err != nil { s.logger.WithError(err).Errorf("[Recover] unable to query order #%d", order.OrderID) - } else if !isActiveOrderBookUpdated { + continue + } + + if !isActiveOrderBookUpdated { s.logger.Infof("[Recover] active order #%d is updated in 3 min, skip updating...", order.OrderID) } + continue } From b41f4712d79c93ad9f1808092427a73943a79dff Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 7 Nov 2023 17:17:38 +0800 Subject: [PATCH 16/33] pkg/exchange: add fee recover --- pkg/exchange/bybit/market_info_poller.go | 10 ++--- pkg/exchange/bybit/market_info_poller_test.go | 12 ++--- pkg/exchange/bybit/stream.go | 44 ++++++++++++++++--- 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/pkg/exchange/bybit/market_info_poller.go b/pkg/exchange/bybit/market_info_poller.go index 6266c8d29..95664718f 100644 --- a/pkg/exchange/bybit/market_info_poller.go +++ b/pkg/exchange/bybit/market_info_poller.go @@ -8,7 +8,6 @@ import ( "time" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" - "github.com/c9s/bbgo/pkg/util" ) const ( @@ -82,15 +81,12 @@ func (p *feeRatePoller) poll(ctx context.Context) error { return nil } -func (p *feeRatePoller) Get(symbol string) (symbolFeeDetail, error) { +func (p *feeRatePoller) Get(symbol string) (symbolFeeDetail, bool) { p.mu.Lock() defer p.mu.Unlock() - fee, ok := p.symbolFeeDetail[symbol] - if !ok { - return symbolFeeDetail{}, fmt.Errorf("%s fee rate not found", symbol) - } - return fee, nil + fee, found := p.symbolFeeDetail[symbol] + return fee, found } func (e *feeRatePoller) getAllFeeRates(ctx context.Context) (map[string]symbolFeeDetail, error) { diff --git a/pkg/exchange/bybit/market_info_poller_test.go b/pkg/exchange/bybit/market_info_poller_test.go index f2b58c466..cf8dd6fdb 100644 --- a/pkg/exchange/bybit/market_info_poller_test.go +++ b/pkg/exchange/bybit/market_info_poller_test.go @@ -137,7 +137,7 @@ func Test_feeRatePoller_Get(t *testing.T) { defer mockCtrl.Finish() mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) - t.Run("succeeds", func(t *testing.T) { + t.Run("found", func(t *testing.T) { symbol := "BTCUSDT" expFeeDetail := symbolFeeDetail{ FeeRate: bybitapi.FeeRate{ @@ -156,18 +156,18 @@ func Test_feeRatePoller_Get(t *testing.T) { }, } - res, err := s.Get(symbol) - assert.NoError(t, err) + res, found := s.Get(symbol) + assert.True(t, found) assert.Equal(t, expFeeDetail, res) }) - t.Run("succeeds", func(t *testing.T) { + t.Run("not found", func(t *testing.T) { symbol := "BTCUSDT" s := &feeRatePoller{ client: mockMarketProvider, symbolFeeDetail: map[string]symbolFeeDetail{}, } - _, err := s.Get(symbol) - assert.ErrorContains(t, err, symbol) + _, found := s.Get(symbol) + assert.False(t, found) }) } diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index 2bb262657..891112567 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -10,6 +10,7 @@ import ( "github.com/gorilla/websocket" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" ) @@ -22,6 +23,11 @@ const ( var ( // wsAuthRequest specifies the duration for which a websocket request's authentication is valid. wsAuthRequest = 10 * time.Second + // The default taker/maker fees can help us in estimating trading fees in the SPOT market, because trade fees are not + // provided for traditional accounts on Bybit. + // https://www.bybit.com/en-US/help-center/article/Trading-Fee-Structure + defaultTakerFee = fixedpoint.NewFromFloat(0.001) + defaultMakerFee = fixedpoint.NewFromFloat(0.001) ) // MarketInfoProvider calculates trade fees since trading fees are not supported by streaming. @@ -48,6 +54,7 @@ type Stream struct { key, secret string streamDataProvider StreamDataProvider feeRateProvider *feeRatePoller + marketsInfo types.MarketMap bookEventCallbacks []func(e BookEvent) marketTradeEventCallbacks []func(e []MarketTradeEvent) @@ -71,8 +78,14 @@ func NewStream(key, secret string, userDataProvider StreamDataProvider) *Stream stream.SetParser(stream.parseWebSocketEvent) stream.SetDispatcher(stream.dispatchEvent) stream.SetHeartBeat(stream.ping) - stream.SetBeforeConnect(func(ctx context.Context) error { + stream.SetBeforeConnect(func(ctx context.Context) (err error) { go stream.feeRateProvider.Start(ctx) + + stream.marketsInfo, err = stream.streamDataProvider.QueryMarkets(ctx) + if err != nil { + log.WithError(err).Error("failed to query market info before to connect stream") + return err + } return nil }) stream.OnConnect(stream.handlerConnect) @@ -406,10 +419,31 @@ func (s *Stream) handleKLineEvent(klineEvent KLineEvent) { func (s *Stream) handleTradeEvent(events []TradeEvent) { for _, event := range events { - feeRate, err := s.feeRateProvider.Get(event.Symbol) - if err != nil { - log.Warnf("failed to get fee rate by symbol: %s", event.Symbol) - continue + feeRate, found := s.feeRateProvider.Get(event.Symbol) + if !found { + feeRate = symbolFeeDetail{ + FeeRate: bybitapi.FeeRate{ + Symbol: event.Symbol, + TakerFeeRate: defaultTakerFee, + MakerFeeRate: defaultMakerFee, + }, + BaseCoin: "", + QuoteCoin: "", + } + + if market, ok := s.marketsInfo[event.Symbol]; ok { + feeRate.BaseCoin = market.BaseCurrency + feeRate.QuoteCoin = market.QuoteCurrency + } + + // The error log level was utilized due to a detected discrepancy in the fee calculations. + log.Errorf("failed to get %s fee rate, use default taker fee %f, maker fee %f, base coin: %s, quote coin: %s", + event.Symbol, + feeRate.TakerFeeRate.Float64(), + feeRate.MakerFeeRate.Float64(), + feeRate.BaseCoin, + feeRate.QuoteCoin, + ) } gTrade, err := event.toGlobalTrade(feeRate) From 52d4f50c883fb945a1968a9d374895ce371f28c8 Mon Sep 17 00:00:00 2001 From: chiahung Date: Wed, 8 Nov 2023 11:15:06 +0800 Subject: [PATCH 17/33] remove sync every ticker --- pkg/strategy/grid2/active_order_recover.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 5c64488b4..c25d45bc4 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -75,7 +75,6 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { case <-ticker.C: s.recoverC <- struct{}{} - bbgo.Sync(ctx, s) case <-s.recoverC: if !time.Now().After(lastRecoverTime.Add(10 * time.Minute)) { continue From 2d650cd1d9766f271ec130e4872c8a5e2e091359 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 8 Nov 2023 22:08:21 +0800 Subject: [PATCH 18/33] pkg/exchange: add defensive program to ensure the order length is expected --- pkg/exchange/bitget/exchange.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index e4eef08a7..542a3bfc2 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -211,10 +211,16 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ orders = append(orders, *order) } - if len(openOrders) != queryOpenOrdersLimit { + orderLen := len(openOrders) + // a defensive programming to ensure the length of order response is expected. + if orderLen > queryOpenOrdersLimit { + return nil, fmt.Errorf("unexpected open orders length %d", orderLen) + } + + if orderLen < queryOpenOrdersLimit { break } - nextCursor = openOrders[len(openOrders)-1].OrderId + nextCursor = openOrders[orderLen-1].OrderId } return orders, nil From 6531dbcf1ae7e95f47a2d6291eb4b2ad0a1e17b3 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 8 Nov 2023 22:18:28 +0800 Subject: [PATCH 19/33] go: update requestgen to v1.3.6 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 91eca5f8b..57f0bf099 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Masterminds/squirrel v1.5.3 github.com/adshao/go-binance/v2 v2.4.2 github.com/c-bata/goptuna v0.8.1 - github.com/c9s/requestgen v1.3.5 + github.com/c9s/requestgen v1.3.6 github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b github.com/cenkalti/backoff/v4 v4.2.0 github.com/cheggaaa/pb/v3 v3.0.8 diff --git a/go.sum b/go.sum index 710eba662..56327767e 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/c9s/requestgen v1.3.4 h1:kK2rIO3OAt9JoY5gT0OSkSpq0dy/+JeuI22FwSKpUrY= github.com/c9s/requestgen v1.3.4/go.mod h1:wp4saiPdh0zLF5AkopGCqPQfy9Q5xvRh+TQBOA1l1r4= github.com/c9s/requestgen v1.3.5 h1:iGYAP0rWQW3JOo+Z3S0SoenSt581IQ9mupJxRFCrCJs= github.com/c9s/requestgen v1.3.5/go.mod h1:QwkZudcv84kJ8g9+E0RDTj+13btFXbTvv2aI+zbuLbc= +github.com/c9s/requestgen v1.3.6 h1:ul7dZ2uwGYjNBjreooRfSY10WTXvQmQSjZsHebz6QfE= +github.com/c9s/requestgen v1.3.6/go.mod h1:QwkZudcv84kJ8g9+E0RDTj+13btFXbTvv2aI+zbuLbc= github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b h1:wT8c03PHLv7+nZUIGqxAzRvIfYHNxMCNVWwvdGkOXTs= github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b/go.mod h1:EKObf66Cp7erWxym2de+07qNN5T1N9PXxHdh97N44EQ= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= From 2c072281d781a1b7819ee279eb746cf7f22ed08e Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 8 Nov 2023 22:43:01 +0800 Subject: [PATCH 20/33] pkg/exchange: add assertion for api response --- .../cancel_order_request_requestgen.go | 20 +++++++++++++++++- pkg/exchange/bybit/bybitapi/client.go | 21 ++++++++++++++++--- .../get_account_info_request_requestgen.go | 20 +++++++++++++++++- .../get_fee_rates_request_requestgen.go | 20 +++++++++++++++++- ...get_instruments_info_request_requestgen.go | 20 +++++++++++++++++- .../get_k_lines_request_requestgen.go | 20 +++++++++++++++++- .../get_open_orders_request_requestgen.go | 20 +++++++++++++++++- ...uest.go => get_order_histories_request.go} | 0 .../get_order_histories_request_requestgen.go | 20 +++++++++++++++++- .../get_tickers_request_requestgen.go | 20 +++++++++++++++++- .../get_wallet_balances_request_requestgen.go | 20 +++++++++++++++++- .../place_order_request_requestgen.go | 20 +++++++++++++++++- .../v3/get_trades_request_requestgen.go | 20 +++++++++++++++++- 13 files changed, 227 insertions(+), 14 deletions(-) rename pkg/exchange/bybit/bybitapi/{get_order_history_request.go => get_order_histories_request.go} (100%) diff --git a/pkg/exchange/bybit/bybitapi/cancel_order_request_requestgen.go b/pkg/exchange/bybit/bybitapi/cancel_order_request_requestgen.go index ad168ded9..715bf4214 100644 --- a/pkg/exchange/bybit/bybitapi/cancel_order_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/cancel_order_request_requestgen.go @@ -187,6 +187,12 @@ func (p *CancelOrderRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (p *CancelOrderRequest) GetPath() string { + return "/v5/order/cancel" +} + +// Do generates the request object and send the request object to the API endpoint func (p *CancelOrderRequest) Do(ctx context.Context) (*CancelOrderResponse, error) { params, err := p.GetParameters() @@ -195,7 +201,9 @@ func (p *CancelOrderRequest) Do(ctx context.Context) (*CancelOrderResponse, erro } query := url.Values{} - apiURL := "/v5/order/cancel" + var apiURL string + + apiURL = p.GetPath() req, err := p.client.NewAuthenticatedRequest(ctx, "POST", apiURL, query, params) if err != nil { @@ -211,6 +219,16 @@ func (p *CancelOrderRequest) Do(ctx context.Context) (*CancelOrderResponse, erro if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data CancelOrderResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/client.go b/pkg/exchange/bybit/bybitapi/client.go index 034d29750..4d9f5818b 100644 --- a/pkg/exchange/bybit/bybitapi/client.go +++ b/pkg/exchange/bybit/bybitapi/client.go @@ -162,10 +162,25 @@ sample: */ type APIResponse struct { - RetCode uint `json:"retCode"` - RetMsg string `json:"retMsg"` - Result json.RawMessage `json:"result"` + // Success/Error code + RetCode uint `json:"retCode"` + // Success/Error msg. OK, success, SUCCESS indicate a successful response + RetMsg string `json:"retMsg"` + // Business data result + Result json.RawMessage `json:"result"` + // Extend info. Most of the time, it is {} RetExtInfo json.RawMessage `json:"retExtInfo"` // Time is current timestamp (ms) Time types.MillisecondTimestamp `json:"time"` } + +func (a APIResponse) Validate() error { + if a.RetCode != 0 { + return a.Error() + } + return nil +} + +func (a APIResponse) Error() error { + return fmt.Errorf("retCode: %d, retMsg: %s, retExtInfo: %q, time: %s", a.RetCode, a.RetMsg, a.RetExtInfo, a.Time) +} diff --git a/pkg/exchange/bybit/bybitapi/get_account_info_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_account_info_request_requestgen.go index 8d9be7df7..c7907f404 100644 --- a/pkg/exchange/bybit/bybitapi/get_account_info_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_account_info_request_requestgen.go @@ -109,13 +109,21 @@ func (g *GetAccountInfoRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetAccountInfoRequest) GetPath() string { + return "/v5/account/info" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetAccountInfoRequest) Do(ctx context.Context) (*AccountInfo, error) { // no body params var params interface{} query := url.Values{} - apiURL := "/v5/account/info" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -131,6 +139,16 @@ func (g *GetAccountInfoRequest) Do(ctx context.Context) (*AccountInfo, error) { if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data AccountInfo if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_fee_rates_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_fee_rates_request_requestgen.go index ac182c22a..e9db73677 100644 --- a/pkg/exchange/bybit/bybitapi/get_fee_rates_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_fee_rates_request_requestgen.go @@ -156,6 +156,12 @@ func (g *GetFeeRatesRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetFeeRatesRequest) GetPath() string { + return "/v5/account/fee-rate" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetFeeRatesRequest) Do(ctx context.Context) (*FeeRates, error) { // no body params @@ -165,7 +171,9 @@ func (g *GetFeeRatesRequest) Do(ctx context.Context) (*FeeRates, error) { return nil, err } - apiURL := "/v5/account/fee-rate" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -181,6 +189,16 @@ func (g *GetFeeRatesRequest) Do(ctx context.Context) (*FeeRates, error) { if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data FeeRates if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_instruments_info_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_instruments_info_request_requestgen.go index 7e9a6536a..ed2fc884f 100644 --- a/pkg/exchange/bybit/bybitapi/get_instruments_info_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_instruments_info_request_requestgen.go @@ -169,6 +169,12 @@ func (g *GetInstrumentsInfoRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetInstrumentsInfoRequest) GetPath() string { + return "/v5/market/instruments-info" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetInstrumentsInfoRequest) Do(ctx context.Context) (*InstrumentsInfo, error) { // no body params @@ -178,7 +184,9 @@ func (g *GetInstrumentsInfoRequest) Do(ctx context.Context) (*InstrumentsInfo, e return nil, err } - apiURL := "/v5/market/instruments-info" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -194,6 +202,16 @@ func (g *GetInstrumentsInfoRequest) Do(ctx context.Context) (*InstrumentsInfo, e if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data InstrumentsInfo if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_k_lines_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_k_lines_request_requestgen.go index d6b3d06c3..e08072a7b 100644 --- a/pkg/exchange/bybit/bybitapi/get_k_lines_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_k_lines_request_requestgen.go @@ -204,6 +204,12 @@ func (g *GetKLinesRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetKLinesRequest) GetPath() string { + return "/v5/market/kline" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetKLinesRequest) Do(ctx context.Context) (*KLinesResponse, error) { // no body params @@ -213,7 +219,9 @@ func (g *GetKLinesRequest) Do(ctx context.Context) (*KLinesResponse, error) { return nil, err } - apiURL := "/v5/market/kline" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -229,6 +237,16 @@ func (g *GetKLinesRequest) Do(ctx context.Context) (*KLinesResponse, error) { if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data KLinesResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_open_orders_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_open_orders_request_requestgen.go index aa9d13cc6..3b7c29ba4 100644 --- a/pkg/exchange/bybit/bybitapi/get_open_orders_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_open_orders_request_requestgen.go @@ -258,6 +258,12 @@ func (g *GetOpenOrdersRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetOpenOrdersRequest) GetPath() string { + return "/v5/order/realtime" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetOpenOrdersRequest) Do(ctx context.Context) (*OrdersResponse, error) { // no body params @@ -267,7 +273,9 @@ func (g *GetOpenOrdersRequest) Do(ctx context.Context) (*OrdersResponse, error) return nil, err } - apiURL := "/v5/order/realtime" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -283,6 +291,16 @@ func (g *GetOpenOrdersRequest) Do(ctx context.Context) (*OrdersResponse, error) if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data OrdersResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_order_history_request.go b/pkg/exchange/bybit/bybitapi/get_order_histories_request.go similarity index 100% rename from pkg/exchange/bybit/bybitapi/get_order_history_request.go rename to pkg/exchange/bybit/bybitapi/get_order_histories_request.go diff --git a/pkg/exchange/bybit/bybitapi/get_order_histories_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_order_histories_request_requestgen.go index 479d9eff4..62b362410 100644 --- a/pkg/exchange/bybit/bybitapi/get_order_histories_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_order_histories_request_requestgen.go @@ -262,6 +262,12 @@ func (g *GetOrderHistoriesRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetOrderHistoriesRequest) GetPath() string { + return "/v5/order/history" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetOrderHistoriesRequest) Do(ctx context.Context) (*OrdersResponse, error) { // no body params @@ -271,7 +277,9 @@ func (g *GetOrderHistoriesRequest) Do(ctx context.Context) (*OrdersResponse, err return nil, err } - apiURL := "/v5/order/history" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -287,6 +295,16 @@ func (g *GetOrderHistoriesRequest) Do(ctx context.Context) (*OrdersResponse, err if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data OrdersResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go index 0f85973d8..59a81a428 100644 --- a/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go @@ -143,6 +143,12 @@ func (g *GetTickersRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetTickersRequest) GetPath() string { + return "/v5/market/tickers" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetTickersRequest) Do(ctx context.Context) (*APIResponse, error) { // no body params @@ -152,7 +158,9 @@ func (g *GetTickersRequest) Do(ctx context.Context) (*APIResponse, error) { return nil, err } - apiURL := "/v5/market/tickers" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -168,5 +176,15 @@ func (g *GetTickersRequest) Do(ctx context.Context) (*APIResponse, error) { if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } return &apiResponse, nil } diff --git a/pkg/exchange/bybit/bybitapi/get_wallet_balances_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_wallet_balances_request_requestgen.go index 14ae4029c..601d26ce9 100644 --- a/pkg/exchange/bybit/bybitapi/get_wallet_balances_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_wallet_balances_request_requestgen.go @@ -143,6 +143,12 @@ func (g *GetWalletBalancesRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetWalletBalancesRequest) GetPath() string { + return "/v5/account/wallet-balance" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetWalletBalancesRequest) Do(ctx context.Context) (*WalletBalancesResponse, error) { // no body params @@ -152,7 +158,9 @@ func (g *GetWalletBalancesRequest) Do(ctx context.Context) (*WalletBalancesRespo return nil, err } - apiURL := "/v5/account/wallet-balance" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -168,6 +176,16 @@ func (g *GetWalletBalancesRequest) Do(ctx context.Context) (*WalletBalancesRespo if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data WalletBalancesResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/place_order_request_requestgen.go b/pkg/exchange/bybit/bybitapi/place_order_request_requestgen.go index bb85937ce..95e3aaa0d 100644 --- a/pkg/exchange/bybit/bybitapi/place_order_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/place_order_request_requestgen.go @@ -496,6 +496,12 @@ func (p *PlaceOrderRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (p *PlaceOrderRequest) GetPath() string { + return "/v5/order/create" +} + +// Do generates the request object and send the request object to the API endpoint func (p *PlaceOrderRequest) Do(ctx context.Context) (*PlaceOrderResponse, error) { params, err := p.GetParameters() @@ -504,7 +510,9 @@ func (p *PlaceOrderRequest) Do(ctx context.Context) (*PlaceOrderResponse, error) } query := url.Values{} - apiURL := "/v5/order/create" + var apiURL string + + apiURL = p.GetPath() req, err := p.client.NewAuthenticatedRequest(ctx, "POST", apiURL, query, params) if err != nil { @@ -520,6 +528,16 @@ func (p *PlaceOrderRequest) Do(ctx context.Context) (*PlaceOrderResponse, error) if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data PlaceOrderResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go b/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go index 00d7dd80a..3336db113 100644 --- a/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go @@ -205,6 +205,12 @@ func (g *GetTradesRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetTradesRequest) GetPath() string { + return "/spot/v3/private/my-trades" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetTradesRequest) Do(ctx context.Context) (*TradesResponse, error) { // no body params @@ -214,7 +220,9 @@ func (g *GetTradesRequest) Do(ctx context.Context) (*TradesResponse, error) { return nil, err } - apiURL := "/spot/v3/private/my-trades" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -230,6 +238,16 @@ func (g *GetTradesRequest) Do(ctx context.Context) (*TradesResponse, error) { if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data TradesResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err From 3978fca27df6b9e524092e93568c8c65a22f8b87 Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 6 Nov 2023 11:51:16 +0800 Subject: [PATCH 21/33] pkg/exchange: support query closed orders --- .../bitget/bitgetapi/v2/client_test.go | 12 +- .../v2/get_history_orders_request.go | 102 ++++++++ .../get_history_orders_request_requestgen.go | 222 ++++++++++++++++++ .../v2/get_history_orders_request_test.go | 121 ++++++++++ pkg/exchange/bitget/convert.go | 85 +++++++ pkg/exchange/bitget/convert_test.go | 193 +++++++++++++++ pkg/exchange/bitget/exchange.go | 64 ++++- 7 files changed, 792 insertions(+), 7 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 21c86dfc6..3d56735e9 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -2,12 +2,11 @@ package bitgetapi import ( "context" + "github.com/stretchr/testify/assert" "os" "strconv" "testing" - "github.com/stretchr/testify/assert" - "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" "github.com/c9s/bbgo/pkg/testutil" ) @@ -25,7 +24,6 @@ func getTestClientOrSkip(t *testing.T) *Client { client := bitgetapi.NewClient() client.Auth(key, secret, os.Getenv("BITGET_API_PASSPHRASE")) - return NewClient(client) } @@ -39,4 +37,12 @@ func TestClient(t *testing.T) { assert.NoError(t, err) t.Logf("resp: %+v", resp) }) + + t.Run("GetHistoryOrdersRequest", func(t *testing.T) { + // market buy + req, err := client.NewGetHistoryOrdersRequest().Symbol("APEUSDT").Do(ctx) + assert.NoError(t, err) + + t.Logf("place order resp: %+v", req) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go new file mode 100644 index 000000000..51807eb5e --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go @@ -0,0 +1,102 @@ +package bitgetapi + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +import ( + "encoding/json" + "fmt" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/requestgen" +) + +type FeeDetail struct { + // NewFees should have a value because when I was integrating, it already prompted, + // "If there is no 'newFees' field, this data represents earlier historical data." + NewFees struct { + // Amount deducted by coupons, unit：currency obtained from the transaction. + DeductedByCoupon fixedpoint.Value `json:"c"` + // Amount deducted in BGB (Bitget Coin), unit：BGB + DeductedInBGB fixedpoint.Value `json:"d"` + // If the BGB balance is insufficient to cover the fees, the remaining amount is deducted from the + //currency obtained from the transaction. + DeductedFromCurrency fixedpoint.Value `json:"r"` + // The total fee amount to be paid, unit ：currency obtained from the transaction. + ToBePaid fixedpoint.Value `json:"t"` + // ignored + Deduction bool `json:"deduction"` + // ignored + TotalDeductionFee fixedpoint.Value `json:"totalDeductionFee"` + } `json:"newFees"` +} + +type OrderDetail struct { + UserId types.StrInt64 `json:"userId"` + Symbol string `json:"symbol"` + // OrderId are always numeric. It's confirmed with official customer service. https://t.me/bitgetOpenapi/24172 + OrderId types.StrInt64 `json:"orderId"` + ClientOrderId string `json:"clientOid"` + Price fixedpoint.Value `json:"price"` + // Size is base coin when orderType=limit; quote coin when orderType=market + Size fixedpoint.Value `json:"size"` + OrderType OrderType `json:"orderType"` + Side SideType `json:"side"` + Status OrderStatus `json:"status"` + PriceAvg fixedpoint.Value `json:"priceAvg"` + BaseVolume fixedpoint.Value `json:"baseVolume"` + QuoteVolume fixedpoint.Value `json:"quoteVolume"` + EnterPointSource string `json:"enterPointSource"` + // The value is json string, so we unmarshal it after unmarshal OrderDetail + FeeDetailRaw string `json:"feeDetail"` + OrderSource string `json:"orderSource"` + CTime types.MillisecondTimestamp `json:"cTime"` + UTime types.MillisecondTimestamp `json:"uTime"` + + FeeDetail FeeDetail +} + +func (o *OrderDetail) UnmarshalJSON(data []byte) error { + if o == nil { + return fmt.Errorf("failed to unmarshal json from nil pointer order detail") + } + // define new type to avoid loop reference + type AuxOrderDetail OrderDetail + + var aux AuxOrderDetail + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + *o = OrderDetail(aux) + + if len(aux.FeeDetailRaw) == 0 { + return nil + } + + var feeDetail FeeDetail + if err := json.Unmarshal([]byte(aux.FeeDetailRaw), &feeDetail); err != nil { + return fmt.Errorf("unexpected fee detail raw: %s, err: %w", aux.FeeDetailRaw, err) + } + o.FeeDetail = feeDetail + + return nil +} + +//go:generate GetRequest -url "/api/v2/spot/trade/history-orders" -type GetHistoryOrdersRequest -responseDataType []OrderDetail +type GetHistoryOrdersRequest struct { + client requestgen.AuthenticatedAPIClient + + symbol *string `param:"symbol,query"` + // Limit number default 100 max 100 + limit *string `param:"limit,query"` + // idLessThan requests the content on the page before this ID (older data), the value input should be the orderId of the corresponding interface. + idLessThan *string `param:"idLessThan,query"` + startTime *int64 `param:"startTime,query"` + endTime *int64 `param:"endTime,query"` + orderId *string `param:"orderId,query"` +} + +func (c *Client) NewGetHistoryOrdersRequest() *GetHistoryOrdersRequest { + return &GetHistoryOrdersRequest{client: c.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go new file mode 100644 index 000000000..0a681f322 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go @@ -0,0 +1,222 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/trade/history-orders -type GetHistoryOrdersRequest -responseDataType []OrderDetail"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (g *GetHistoryOrdersRequest) Symbol(symbol string) *GetHistoryOrdersRequest { + g.symbol = &symbol + return g +} + +func (g *GetHistoryOrdersRequest) Limit(limit string) *GetHistoryOrdersRequest { + g.limit = &limit + return g +} + +func (g *GetHistoryOrdersRequest) IdLessThan(idLessThan string) *GetHistoryOrdersRequest { + g.idLessThan = &idLessThan + return g +} + +func (g *GetHistoryOrdersRequest) StartTime(startTime int64) *GetHistoryOrdersRequest { + g.startTime = &startTime + return g +} + +func (g *GetHistoryOrdersRequest) EndTime(endTime int64) *GetHistoryOrdersRequest { + g.endTime = &endTime + return g +} + +func (g *GetHistoryOrdersRequest) OrderId(orderId string) *GetHistoryOrdersRequest { + g.orderId = &orderId + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetHistoryOrdersRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + if g.symbol != nil { + symbol := *g.symbol + + // assign parameter of symbol + params["symbol"] = symbol + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + // check idLessThan field -> json key idLessThan + if g.idLessThan != nil { + idLessThan := *g.idLessThan + + // assign parameter of idLessThan + params["idLessThan"] = idLessThan + } else { + } + // check startTime field -> json key startTime + if g.startTime != nil { + startTime := *g.startTime + + // assign parameter of startTime + params["startTime"] = startTime + } else { + } + // check endTime field -> json key endTime + if g.endTime != nil { + endTime := *g.endTime + + // assign parameter of endTime + params["endTime"] = endTime + } else { + } + // check orderId field -> json key orderId + if g.orderId != nil { + orderId := *g.orderId + + // assign parameter of orderId + params["orderId"] = orderId + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetHistoryOrdersRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetHistoryOrdersRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetHistoryOrdersRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetHistoryOrdersRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetHistoryOrdersRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetHistoryOrdersRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetHistoryOrdersRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetHistoryOrdersRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (g *GetHistoryOrdersRequest) Do(ctx context.Context) ([]OrderDetail, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + apiURL := "/api/v2/spot/trade/history-orders" + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + var data []OrderDetail + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go new file mode 100644 index 000000000..07e319fab --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go @@ -0,0 +1,121 @@ +package bitgetapi + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func TestOrderDetail_UnmarshalJSON(t *testing.T) { + var ( + assert = assert.New(t) + ) + t.Run("empty fee", func(t *testing.T) { + input := `{ + "userId":"8672173294", + "symbol":"APEUSDT", + "orderId":"1104342023170068480", + "clientOid":"f3d6a1ee-4e94-48b5-a6e0-25f3e93d92e1", + "price":"1.2000000000000000", + "size":"5.0000000000000000", + "orderType":"limit", + "side":"buy", + "status":"cancelled", + "priceAvg":"0", + "baseVolume":"0.0000000000000000", + "quoteVolume":"0.0000000000000000", + "enterPointSource":"API", + "feeDetail":"", + "orderSource":"normal", + "cTime":"1699021576683", + "uTime":"1699021649099" + }` + var od OrderDetail + err := json.Unmarshal([]byte(input), &od) + assert.NoError(err) + assert.Equal(OrderDetail{ + UserId: types.StrInt64(8672173294), + Symbol: "APEUSDT", + OrderId: types.StrInt64(1104342023170068480), + ClientOrderId: "f3d6a1ee-4e94-48b5-a6e0-25f3e93d92e1", + Price: fixedpoint.NewFromFloat(1.2), + Size: fixedpoint.NewFromFloat(5), + OrderType: OrderTypeLimit, + Side: SideTypeBuy, + Status: OrderStatusCancelled, + PriceAvg: fixedpoint.Zero, + BaseVolume: fixedpoint.Zero, + QuoteVolume: fixedpoint.Zero, + EnterPointSource: "API", + FeeDetailRaw: "", + OrderSource: "normal", + CTime: types.NewMillisecondTimestampFromInt(1699021576683), + UTime: types.NewMillisecondTimestampFromInt(1699021649099), + FeeDetail: FeeDetail{}, + }, od) + }) + + t.Run("fee", func(t *testing.T) { + input := `{ + "userId":"8672173294", + "symbol":"APEUSDT", + "orderId":"1104337778433757184", + "clientOid":"8afea7bd-d873-44fe-aff8-6a1fae3cc765", + "price":"1.4000000000000000", + "size":"5.0000000000000000", + "orderType":"limit", + "side":"sell", + "status":"filled", + "priceAvg":"1.4001000000000000", + "baseVolume":"5.0000000000000000", + "quoteVolume":"7.0005000000000000", + "enterPointSource":"API", + "feeDetail":"{\"newFees\":{\"c\":0,\"d\":0,\"deduction\":false,\"r\":-0.0070005,\"t\":-0.0070005,\"totalDeductionFee\":0},\"USDT\":{\"deduction\":false,\"feeCoinCode\":\"USDT\",\"totalDeductionFee\":0,\"totalFee\":-0.007000500000}}", + "orderSource":"normal", + "cTime":"1699020564659", + "uTime":"1699020564688" + }` + var od OrderDetail + err := json.Unmarshal([]byte(input), &od) + assert.NoError(err) + assert.Equal(OrderDetail{ + UserId: types.StrInt64(8672173294), + Symbol: "APEUSDT", + OrderId: types.StrInt64(1104337778433757184), + ClientOrderId: "8afea7bd-d873-44fe-aff8-6a1fae3cc765", + Price: fixedpoint.NewFromFloat(1.4), + Size: fixedpoint.NewFromFloat(5), + OrderType: OrderTypeLimit, + Side: SideTypeSell, + Status: OrderStatusFilled, + PriceAvg: fixedpoint.NewFromFloat(1.4001), + BaseVolume: fixedpoint.NewFromFloat(5), + QuoteVolume: fixedpoint.NewFromFloat(7.0005), + EnterPointSource: "API", + FeeDetailRaw: `{"newFees":{"c":0,"d":0,"deduction":false,"r":-0.0070005,"t":-0.0070005,"totalDeductionFee":0},"USDT":{"deduction":false,"feeCoinCode":"USDT","totalDeductionFee":0,"totalFee":-0.007000500000}}`, + OrderSource: "normal", + CTime: types.NewMillisecondTimestampFromInt(1699020564659), + UTime: types.NewMillisecondTimestampFromInt(1699020564688), + FeeDetail: FeeDetail{ + NewFees: struct { + DeductedByCoupon fixedpoint.Value `json:"c"` + DeductedInBGB fixedpoint.Value `json:"d"` + DeductedFromCurrency fixedpoint.Value `json:"r"` + ToBePaid fixedpoint.Value `json:"t"` + Deduction bool `json:"deduction"` + TotalDeductionFee fixedpoint.Value `json:"totalDeductionFee"` + }{DeductedByCoupon: fixedpoint.NewFromFloat(0), + DeductedInBGB: fixedpoint.NewFromFloat(0), + DeductedFromCurrency: fixedpoint.NewFromFloat(-0.0070005), + ToBePaid: fixedpoint.NewFromFloat(-0.0070005), + Deduction: false, + TotalDeductionFee: fixedpoint.Zero, + }, + }, + }, od) + }) +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index a4a9756d0..24785a81f 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -1,6 +1,7 @@ package bitget import ( + "errors" "fmt" "math" "strconv" @@ -158,3 +159,87 @@ func unfilledOrderToGlobalOrder(order v2.UnfilledOrder) (*types.Order, error) { UpdateTime: types.Time(order.UTime.Time()), }, nil } + +func toGlobalOrder(order v2.OrderDetail) (*types.Order, error) { + side, err := toGlobalSideType(order.Side) + if err != nil { + return nil, err + } + + orderType, err := toGlobalOrderType(order.OrderType) + if err != nil { + return nil, err + } + + status, err := toGlobalOrderStatus(order.Status) + if err != nil { + return nil, err + } + + qty := order.Size + price := order.Price + + if orderType == types.OrderTypeMarket { + price = order.PriceAvg + if side == types.SideTypeBuy { + qty, err = processMarketBuyQuantity(order.BaseVolume, order.QuoteVolume, order.PriceAvg, order.Size, order.Status) + if err != nil { + return nil, err + } + } + } + + return &types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: order.ClientOrderId, + Symbol: order.Symbol, + Side: side, + Type: orderType, + Quantity: qty, + Price: price, + // Bitget does not include the "time-in-force" field in its API response for spot trading, so we set GTC. + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(order.OrderId), + UUID: strconv.FormatInt(int64(order.OrderId), 10), + Status: status, + ExecutedQuantity: order.BaseVolume, + IsWorking: order.Status.IsWorking(), + CreationTime: types.Time(order.CTime.Time()), + UpdateTime: types.Time(order.UTime.Time()), + }, nil +} + +// processMarketBuyQuantity returns the estimated base quantity or real. The order size will be 'quote quantity' when side is buy and +// type is market, so we need to convert that. This is because the unit of types.Order.Quantity is base coin. +// +// If the order status is PartialFilled, return estimated base coin quantity. +// If the order status is Filled, return the filled base quantity instead of the buy quantity, because a market order on the buy side +// cannot execute all. +// Otherwise, return zero. +func processMarketBuyQuantity(filledQty, filledPrice, priceAvg, buyQty fixedpoint.Value, orderStatus v2.OrderStatus) (fixedpoint.Value, error) { + switch orderStatus { + case v2.OrderStatusInit, v2.OrderStatusNew, v2.OrderStatusLive, v2.OrderStatusCancelled: + return fixedpoint.Zero, nil + + case v2.OrderStatusPartialFilled: + // sanity check for avoid divide 0 + if priceAvg.IsZero() { + return fixedpoint.Zero, errors.New("priceAvg for a partialFilled should not be zero") + } + // calculate the remaining quote coin quantity. + remainPrice := buyQty.Sub(filledPrice) + // calculate the remaining base coin quantity. + remainBaseCoinQty := remainPrice.Div(priceAvg) + // Estimated quantity that may be purchased. + return filledQty.Add(remainBaseCoinQty), nil + + case v2.OrderStatusFilled: + // Market buy orders may not purchase the entire quantity, hence the use of filledQty here. + return filledQty, nil + + default: + return fixedpoint.Zero, fmt.Errorf("failed to execute market buy quantity due to unexpected order status %s ", orderStatus) + } +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 8b8bc61c7..174798074 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -272,3 +272,196 @@ func Test_unfilledOrderToGlobalOrder(t *testing.T) { assert.ErrorContains(err, "xxx") }) } + +func Test_toGlobalOrder(t *testing.T) { + var ( + assert = assert.New(t) + orderId = 1105087175647989764 + unfilledOrder = v2.OrderDetail{ + UserId: 123456, + Symbol: "BTCUSDT", + OrderId: types.StrInt64(orderId), + ClientOrderId: "74b86af3-6098-479c-acac-bfb074c067f3", + Price: fixedpoint.NewFromFloat(1.2), + Size: fixedpoint.NewFromFloat(5), + OrderType: v2.OrderTypeLimit, + Side: v2.SideTypeBuy, + Status: v2.OrderStatusFilled, + PriceAvg: fixedpoint.NewFromFloat(1.4), + BaseVolume: fixedpoint.NewFromFloat(5), + QuoteVolume: fixedpoint.NewFromFloat(7.0005), + EnterPointSource: "API", + FeeDetailRaw: `{\"newFees\":{\"c\":0,\"d\":0,\"deduction\":false,\"r\":-0.0070005,\"t\":-0.0070005,\"totalDeductionFee\":0},\"USDT\":{\"deduction\":false,\"feeCoinCode\":\"USDT\",\"totalDeductionFee\":0,\"totalFee\":-0.007000500000}}`, + OrderSource: "normal", + CTime: types.NewMillisecondTimestampFromInt(1660704288118), + UTime: types.NewMillisecondTimestampFromInt(1660704288118), + } + + expOrder = &types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: "74b86af3-6098-479c-acac-bfb074c067f3", + Symbol: "BTCUSDT", + Side: types.SideTypeBuy, + Type: types.OrderTypeLimit, + Quantity: fixedpoint.NewFromFloat(5), + Price: fixedpoint.NewFromFloat(1.2), + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(orderId), + UUID: strconv.FormatInt(int64(orderId), 10), + Status: types.OrderStatusFilled, + ExecutedQuantity: fixedpoint.NewFromFloat(5), + IsWorking: false, + CreationTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + UpdateTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + } + ) + + t.Run("succeeds with limit buy", func(t *testing.T) { + order, err := toGlobalOrder(unfilledOrder) + assert.NoError(err) + assert.Equal(expOrder, order) + }) + + t.Run("succeeds with limit sell", func(t *testing.T) { + newUnfilledOrder := unfilledOrder + newUnfilledOrder.Side = v2.SideTypeSell + + newExpOrder := *expOrder + newExpOrder.Side = types.SideTypeSell + + order, err := toGlobalOrder(newUnfilledOrder) + assert.NoError(err) + assert.Equal(&newExpOrder, order) + }) + + t.Run("succeeds with market sell", func(t *testing.T) { + newUnfilledOrder := unfilledOrder + newUnfilledOrder.Side = v2.SideTypeSell + newUnfilledOrder.OrderType = v2.OrderTypeMarket + + newExpOrder := *expOrder + newExpOrder.Side = types.SideTypeSell + newExpOrder.Type = types.OrderTypeMarket + newExpOrder.Price = newUnfilledOrder.PriceAvg + + order, err := toGlobalOrder(newUnfilledOrder) + assert.NoError(err) + assert.Equal(&newExpOrder, order) + }) + + t.Run("succeeds with market buy", func(t *testing.T) { + newUnfilledOrder := unfilledOrder + newUnfilledOrder.Side = v2.SideTypeBuy + newUnfilledOrder.OrderType = v2.OrderTypeMarket + + newExpOrder := *expOrder + newExpOrder.Side = types.SideTypeBuy + newExpOrder.Type = types.OrderTypeMarket + newExpOrder.Price = newUnfilledOrder.PriceAvg + newExpOrder.Quantity = newUnfilledOrder.BaseVolume + + order, err := toGlobalOrder(newUnfilledOrder) + assert.NoError(err) + assert.Equal(&newExpOrder, order) + }) + + t.Run("succeeds with limit buy", func(t *testing.T) { + order, err := toGlobalOrder(unfilledOrder) + assert.NoError(err) + assert.Equal(&types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: "74b86af3-6098-479c-acac-bfb074c067f3", + Symbol: "BTCUSDT", + Side: types.SideTypeBuy, + Type: types.OrderTypeLimit, + Quantity: fixedpoint.NewFromFloat(5), + Price: fixedpoint.NewFromFloat(1.2), + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(orderId), + UUID: strconv.FormatInt(int64(orderId), 10), + Status: types.OrderStatusFilled, + ExecutedQuantity: fixedpoint.NewFromFloat(5), + IsWorking: false, + CreationTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + UpdateTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + }, order) + }) + + t.Run("failed to convert side", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.Side = "xxx" + + _, err := toGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) + + t.Run("failed to convert oder type", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.OrderType = "xxx" + + _, err := toGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) + + t.Run("failed to convert oder status", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.Status = "xxx" + + _, err := toGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) +} + +func Test_processMarketBuyQuantity(t *testing.T) { + var ( + assert = assert.New(t) + filledBaseCoinQty = fixedpoint.NewFromFloat(3.5648) + filledPrice = fixedpoint.NewFromFloat(4.99998848) + priceAvg = fixedpoint.NewFromFloat(1.4026) + buyQty = fixedpoint.NewFromFloat(5) + ) + + t.Run("zero quantity on Init/New/Live/Cancelled", func(t *testing.T) { + qty, err := processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusInit) + assert.NoError(err) + assert.Equal(fixedpoint.Zero, qty) + + qty, err = processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusNew) + assert.NoError(err) + assert.Equal(fixedpoint.Zero, qty) + + qty, err = processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusLive) + assert.NoError(err) + assert.Equal(fixedpoint.Zero, qty) + + qty, err = processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusCancelled) + assert.NoError(err) + assert.Equal(fixedpoint.Zero, qty) + }) + + t.Run("5 on PartialFilled", func(t *testing.T) { + priceAvg := fixedpoint.NewFromFloat(2) + buyQty := fixedpoint.NewFromFloat(10) + filledPrice := fixedpoint.NewFromFloat(4) + filledBaseCoinQty := fixedpoint.NewFromFloat(2) + + qty, err := processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusPartialFilled) + assert.NoError(err) + assert.Equal(fixedpoint.NewFromFloat(5), qty) + }) + + t.Run("3.5648 on Filled", func(t *testing.T) { + qty, err := processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusFilled) + assert.NoError(err) + assert.Equal(fixedpoint.NewFromFloat(3.5648), qty) + }) + + t.Run("unexpected order status", func(t *testing.T) { + _, err := processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, "xxx") + assert.ErrorContains(err, "xxx") + }) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 542a3bfc2..298054e0a 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -7,6 +7,7 @@ import ( "time" "github.com/sirupsen/logrus" + "go.uber.org/multierr" "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" @@ -19,7 +20,9 @@ const ( PlatformToken = "BGB" - queryOpenOrdersLimit = 100 + queryLimit = 100 + maxOrderIdLen = 36 + queryMaxDuration = 90 * 24 * time.Hour ) var log = logrus.WithFields(logrus.Fields{ @@ -37,6 +40,8 @@ var ( queryTickersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) // queryOpenOrdersRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Get-Unfilled-Orders queryOpenOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // closedQueryOrdersRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/trade/Get-History-Orders + closedQueryOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/15), 5) ) type Exchange struct { @@ -192,7 +197,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ req := e.v2Client.NewGetUnfilledOrdersRequest(). Symbol(symbol). - Limit(strconv.FormatInt(queryOpenOrdersLimit, 10)) + Limit(strconv.FormatInt(queryLimit, 10)) if nextCursor != 0 { req.IdLessThan(strconv.FormatInt(int64(nextCursor), 10)) } @@ -213,11 +218,11 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ orderLen := len(openOrders) // a defensive programming to ensure the length of order response is expected. - if orderLen > queryOpenOrdersLimit { + if orderLen > queryLimit { return nil, fmt.Errorf("unexpected open orders length %d", orderLen) } - if orderLen < queryOpenOrdersLimit { + if orderLen < queryLimit { break } nextCursor = openOrders[orderLen-1].OrderId @@ -226,6 +231,57 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ return orders, nil } +// QueryClosedOrders queries closed order by time range(`CTime`) and id. The order of the response is in descending order. +// If you need to retrieve all data, please utilize the function pkg/exchange/batch.ClosedOrderBatchQuery. +// +// ** Since is inclusive, Until is exclusive. If you use a time range to query, you must provide both a start time and an end time. ** +// ** Since and Until cannot exceed 90 days. ** +// ** Since from the last 90 days can be queried. ** +func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) { + if since.Sub(time.Now()) > queryMaxDuration { + return nil, fmt.Errorf("start time from the last 90 days can be queried, got: %s", since) + } + if until.Before(since) { + return nil, fmt.Errorf("end time %s before start %s", until, since) + } + if until.Sub(since) > queryMaxDuration { + return nil, fmt.Errorf("the start time %s and end time %s cannot exceed 90 days", since, until) + } + if lastOrderID != 0 { + log.Warn("!!!BITGET EXCHANGE API NOTICE!!! The order of response is in descending order, so the last order id not supported.") + } + + if err := closedQueryOrdersRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("query closed order rate limiter wait error: %w", err) + } + res, err := e.v2Client.NewGetHistoryOrdersRequest(). + Symbol(symbol). + Limit(strconv.Itoa(queryLimit)). + StartTime(since.UnixMilli()). + EndTime(until.UnixMilli()). + Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call get order histories error: %w", err) + } + + for _, order := range res { + o, err2 := toGlobalOrder(order) + if err2 != nil { + err = multierr.Append(err, err2) + continue + } + + if o.Status.Closed() { + orders = append(orders, *o) + } + } + if err != nil { + return nil, err + } + + return types.SortOrdersAscending(orders), nil +} + func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) error { // TODO implement me panic("implement me") From 2c842e54e8f5513ed98aee2eb2f966fee171349a Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 1 Nov 2023 17:01:04 +0800 Subject: [PATCH 22/33] scmaker: fix scmaker stream book binding --- pkg/strategy/scmaker/strategy.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/strategy/scmaker/strategy.go b/pkg/strategy/scmaker/strategy.go index ecccefc8b..d9eb8d7aa 100644 --- a/pkg/strategy/scmaker/strategy.go +++ b/pkg/strategy/scmaker/strategy.go @@ -19,8 +19,6 @@ import ( const ID = "scmaker" -var ten = fixedpoint.NewFromInt(10) - type advancedOrderCancelApi interface { CancelAllOrders(ctx context.Context) ([]types.Order, error) CancelOrdersBySymbol(ctx context.Context, symbol string) ([]types.Order, error) @@ -100,12 +98,12 @@ func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { } } -func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { +func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) s.book = types.NewStreamBook(s.Symbol) - s.book.BindStream(session.UserDataStream) + s.book.BindStream(session.MarketDataStream) s.liquidityOrderBook = bbgo.NewActiveOrderBook(s.Symbol) s.liquidityOrderBook.BindStream(session.UserDataStream) @@ -174,7 +172,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se return nil } -func (s *Strategy) preloadKLines(inc *KLineStream, session *bbgo.ExchangeSession, symbol string, interval types.Interval) { +func (s *Strategy) preloadKLines( + inc *KLineStream, session *bbgo.ExchangeSession, symbol string, interval types.Interval, +) { if store, ok := session.MarketDataStore(symbol); ok { if kLinesData, ok := store.KLinesOfInterval(interval); ok { for _, k := range *kLinesData { @@ -476,7 +476,9 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { log.Infof("%d liq orders are placed successfully", len(liqOrders)) } -func profitProtectedPrice(side types.SideType, averageCost, price, feeRate, minProfit fixedpoint.Value) fixedpoint.Value { +func profitProtectedPrice( + side types.SideType, averageCost, price, feeRate, minProfit fixedpoint.Value, +) fixedpoint.Value { switch side { case types.SideTypeSell: minProfitPrice := averageCost.Add( From d2dab58193009367a73977c77d770b23cee25f86 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 1 Nov 2023 17:05:10 +0800 Subject: [PATCH 23/33] scmaker: clean up scmaker risk control --- pkg/strategy/scmaker/strategy.go | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/pkg/strategy/scmaker/strategy.go b/pkg/strategy/scmaker/strategy.go index d9eb8d7aa..79e3f8956 100644 --- a/pkg/strategy/scmaker/strategy.go +++ b/pkg/strategy/scmaker/strategy.go @@ -5,14 +5,12 @@ import ( "fmt" "math" "sync" - "time" log "github.com/sirupsen/logrus" "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" . "github.com/c9s/bbgo/pkg/indicator/v2" - "github.com/c9s/bbgo/pkg/risk/riskcontrol" "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" ) @@ -60,12 +58,6 @@ type Strategy struct { MinProfit fixedpoint.Value `json:"minProfit"` - // risk related parameters - PositionHardLimit fixedpoint.Value `json:"positionHardLimit"` - MaxPositionQuantity fixedpoint.Value `json:"maxPositionQuantity"` - CircuitBreakLossThreshold fixedpoint.Value `json:"circuitBreakLossThreshold"` - CircuitBreakEMA types.IntervalWindow `json:"circuitBreakEMA"` - liquidityOrderBook, adjustmentOrderBook *bbgo.ActiveOrderBook book *types.StreamOrderBook @@ -75,9 +67,6 @@ type Strategy struct { ewma *EWMAStream boll *BOLLStream intensity *IntensityStream - - positionRiskControl *riskcontrol.PositionRiskControl - circuitBreakRiskControl *riskcontrol.CircuitBreakRiskControl } func (s *Strategy) ID() string { @@ -111,21 +100,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.adjustmentOrderBook = bbgo.NewActiveOrderBook(s.Symbol) s.adjustmentOrderBook.BindStream(session.UserDataStream) - if !s.PositionHardLimit.IsZero() && !s.MaxPositionQuantity.IsZero() { - log.Infof("positionHardLimit and maxPositionQuantity are configured, setting up PositionRiskControl...") - s.positionRiskControl = riskcontrol.NewPositionRiskControl(s.OrderExecutor, s.PositionHardLimit, s.MaxPositionQuantity) - } - - if !s.CircuitBreakLossThreshold.IsZero() { - log.Infof("circuitBreakLossThreshold is configured, setting up CircuitBreakRiskControl...") - s.circuitBreakRiskControl = riskcontrol.NewCircuitBreakRiskControl( - s.Position, - session.Indicators(s.Symbol).EWMA(s.CircuitBreakEMA), - s.CircuitBreakLossThreshold, - s.ProfitStats, - 24*time.Hour) - } - scale, err := s.LiquiditySlideRule.Scale() if err != nil { return err @@ -282,7 +256,7 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { return } - if s.circuitBreakRiskControl != nil && s.circuitBreakRiskControl.IsHalted(ticker.Time) { + if s.IsHalted(ticker.Time) { log.Warn("circuitBreakRiskControl: trading halted") return } From dda2cfb73de27f6597cf924bf6fbfdf4b92ea47a Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 2 Nov 2023 11:59:47 +0800 Subject: [PATCH 24/33] liquiditymaker: first commit --- pkg/strategy/liquiditymaker/strategy.go | 440 ++++++++++++++++++++++++ 1 file changed, 440 insertions(+) create mode 100644 pkg/strategy/liquiditymaker/strategy.go diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go new file mode 100644 index 000000000..fcd8f17de --- /dev/null +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -0,0 +1,440 @@ +package liquiditymaker + +import ( + "context" + "fmt" + "math" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + . "github.com/c9s/bbgo/pkg/indicator/v2" + "github.com/c9s/bbgo/pkg/strategy/common" + "github.com/c9s/bbgo/pkg/types" +) + +const ID = "liquiditymaker" + +type advancedOrderCancelApi interface { + CancelAllOrders(ctx context.Context) ([]types.Order, error) + CancelOrdersBySymbol(ctx context.Context, symbol string) ([]types.Order, error) +} + +func init() { + bbgo.RegisterStrategy(ID, &Strategy{}) +} + +// Strategy is the strategy struct of LiquidityMaker +// liquidity maker does not care about the current price, it tries to place liquidity orders (limit maker orders) +// around the current mid price +// liquidity maker's target: +// - place enough total liquidity amount on the order book, for example, 20k USDT value liquidity on both sell and buy +// - ensure the spread by placing the orders from the mid price (or the last trade price) +type Strategy struct { + *common.Strategy + + Environment *bbgo.Environment + Market types.Market + + Symbol string `json:"symbol"` + + LiquidityUpdateInterval types.Interval `json:"liquidityUpdateInterval"` + + AdjustmentUpdateInterval types.Interval `json:"adjustmentUpdateInterval"` + + NumOfLiquidityLayers int `json:"numOfLiquidityLayers"` + LiquiditySlideRule *bbgo.SlideRule `json:"liquidityScale"` + LiquidityLayerTickSize fixedpoint.Value `json:"liquidityLayerTickSize"` + LiquiditySkew fixedpoint.Value `json:"liquiditySkew"` + LiquidityPriceRange fixedpoint.Value `json:"liquidityPriceRange"` + + Spread fixedpoint.Value `json:"spread"` + MaxPrice fixedpoint.Value `json:"maxPrice"` + MinPrice fixedpoint.Value `json:"minPrice"` + + MaxExposure fixedpoint.Value `json:"maxExposure"` + + MinProfit fixedpoint.Value `json:"minProfit"` + + liquidityOrderBook, adjustmentOrderBook *bbgo.ActiveOrderBook + book *types.StreamOrderBook + + liquidityScale bbgo.Scale +} + +func (s *Strategy) ID() string { + return ID +} + +func (s *Strategy) InstanceID() string { + return fmt.Sprintf("%s:%s", ID, s.Symbol) +} + +func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { + session.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{}) + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.AdjustmentUpdateInterval}) + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.LiquidityUpdateInterval}) +} + +func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { + s.Strategy = &common.Strategy{} + s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) + + s.book = types.NewStreamBook(s.Symbol) + s.book.BindStream(session.MarketDataStream) + + s.liquidityOrderBook = bbgo.NewActiveOrderBook(s.Symbol) + s.liquidityOrderBook.BindStream(session.UserDataStream) + + s.adjustmentOrderBook = bbgo.NewActiveOrderBook(s.Symbol) + s.adjustmentOrderBook.BindStream(session.UserDataStream) + + scale, err := s.LiquiditySlideRule.Scale() + if err != nil { + return err + } + + if err := scale.Solve(); err != nil { + return err + } + + if cancelApi, ok := session.Exchange.(advancedOrderCancelApi); ok { + _, _ = cancelApi.CancelOrdersBySymbol(ctx, s.Symbol) + } + + s.liquidityScale = scale + + session.UserDataStream.OnStart(func() { + s.placeLiquidityOrders(ctx) + }) + + session.MarketDataStream.OnKLineClosed(func(k types.KLine) { + if k.Interval == s.AdjustmentUpdateInterval { + s.placeAdjustmentOrders(ctx) + } + + if k.Interval == s.LiquidityUpdateInterval { + s.placeLiquidityOrders(ctx) + } + }) + + bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + + if err := s.liquidityOrderBook.GracefulCancel(ctx, s.Session.Exchange); err != nil { + logErr(err, "unable to cancel liquidity orders") + } + + if err := s.adjustmentOrderBook.GracefulCancel(ctx, s.Session.Exchange); err != nil { + logErr(err, "unable to cancel adjustment orders") + } + }) + + return nil +} + +func (s *Strategy) placeAdjustmentOrders(ctx context.Context) { + _ = s.adjustmentOrderBook.GracefulCancel(ctx, s.Session.Exchange) + + if s.Position.IsDust() { + return + } + + ticker, err := s.Session.Exchange.QueryTicker(ctx, s.Symbol) + if logErr(err, "unable to query ticker") { + return + } + + if _, err := s.Session.UpdateAccount(ctx); err != nil { + logErr(err, "unable to update account") + return + } + + baseBal, _ := s.Session.Account.Balance(s.Market.BaseCurrency) + quoteBal, _ := s.Session.Account.Balance(s.Market.QuoteCurrency) + + var adjOrders []types.SubmitOrder + + posSize := s.Position.Base.Abs() + tickSize := s.Market.TickSize + + if s.Position.IsShort() { + price := profitProtectedPrice(types.SideTypeBuy, s.Position.AverageCost, ticker.Sell.Add(tickSize.Neg()), s.Session.MakerFeeRate, s.MinProfit) + quoteQuantity := fixedpoint.Min(price.Mul(posSize), quoteBal.Available) + bidQuantity := quoteQuantity.Div(price) + + if s.Market.IsDustQuantity(bidQuantity, price) { + return + } + + adjOrders = append(adjOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Type: types.OrderTypeLimitMaker, + Side: types.SideTypeBuy, + Price: price, + Quantity: bidQuantity, + Market: s.Market, + TimeInForce: types.TimeInForceGTC, + }) + } else if s.Position.IsLong() { + price := profitProtectedPrice(types.SideTypeSell, s.Position.AverageCost, ticker.Buy.Add(tickSize), s.Session.MakerFeeRate, s.MinProfit) + askQuantity := fixedpoint.Min(posSize, baseBal.Available) + + if s.Market.IsDustQuantity(askQuantity, price) { + return + } + + adjOrders = append(adjOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Type: types.OrderTypeLimitMaker, + Side: types.SideTypeSell, + Price: price, + Quantity: askQuantity, + Market: s.Market, + TimeInForce: types.TimeInForceGTC, + }) + } + + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, adjOrders...) + if logErr(err, "unable to place liquidity orders") { + return + } + + s.adjustmentOrderBook.Add(createdOrders...) +} + +func (s *Strategy) placeLiquidityOrders(ctx context.Context) { + ticker, err := s.Session.Exchange.QueryTicker(ctx, s.Symbol) + if logErr(err, "unable to query ticker") { + return + } + + if s.IsHalted(ticker.Time) { + log.Warn("circuitBreakRiskControl: trading halted") + return + } + + err = s.liquidityOrderBook.GracefulCancel(ctx, s.Session.Exchange) + if logErr(err, "unable to cancel orders") { + return + } + + if ticker.Buy.IsZero() && ticker.Sell.IsZero() { + ticker.Sell = ticker.Last.Add(s.Market.TickSize) + ticker.Buy = ticker.Last.Sub(s.Market.TickSize) + } else if ticker.Buy.IsZero() { + ticker.Buy = ticker.Sell.Sub(s.Market.TickSize) + } else if ticker.Sell.IsZero() { + ticker.Sell = ticker.Buy.Add(s.Market.TickSize) + } + + if _, err := s.Session.UpdateAccount(ctx); err != nil { + logErr(err, "unable to update account") + return + } + + baseBal, _ := s.Session.Account.Balance(s.Market.BaseCurrency) + quoteBal, _ := s.Session.Account.Balance(s.Market.QuoteCurrency) + + lastTradedPrice := ticker.Last + midPrice := ticker.Sell.Add(ticker.Buy).Div(fixedpoint.Two) + currentSpread := ticker.Sell.Sub(ticker.Buy) + tickSize := fixedpoint.Max(s.LiquidityLayerTickSize, s.Market.TickSize) + sideSpread := s.Spread.Div(fixedpoint.Two) + + log.Infof("current: spread: %f lastTradedPrice: %f midPrice: %f", currentSpread.Float64(), lastTradedPrice.Float64(), midPrice.Float64()) + + ask1Price := midPrice.Mul(fixedpoint.One.Add(sideSpread)) + bid1Price := midPrice.Mul(fixedpoint.One.Sub(sideSpread)) + + askLastPrice := midPrice.Mul(fixedpoint.One.Add(s.LiquidityPriceRange)) + bidLastPrice := midPrice.Mul(fixedpoint.One.Sub(s.LiquidityPriceRange)) + log.Infof("wanted side spread: %f askRange: %f ~ %f bidRange: %f ~ %f", sideSpread.Float64(), + ask1Price.Float64(), askLastPrice.Float64(), + bid1Price.Float64(), bidLastPrice.Float64()) + + askLayerSpread := askLastPrice.Sub(ask1Price).Div(fixedpoint.NewFromInt(int64(s.NumOfLiquidityLayers))) + bidLayerSpread := bid1Price.Sub(bidLastPrice).Div(fixedpoint.NewFromInt(int64(s.NumOfLiquidityLayers))) + + if askLayerSpread.Compare(tickSize) < 0 { + askLayerSpread = tickSize + } + + if bidLayerSpread.Compare(tickSize) < 0 { + bidLayerSpread = tickSize + } + + sum := s.liquidityScale.Sum(1.0) + askSum := sum + bidSum := sum + log.Infof("liquidity sum: %f / %f", askSum, bidSum) + + skew := s.LiquiditySkew.Float64() + useSkew := !s.LiquiditySkew.IsZero() + if useSkew { + askSum = sum / skew + bidSum = sum * skew + log.Infof("adjusted liqudity skew: %f / %f", askSum, bidSum) + } + + var bidPrices []fixedpoint.Value + var askPrices []fixedpoint.Value + + // calculate and collect prices + for i := 0; i <= s.NumOfLiquidityLayers; i++ { + fi := fixedpoint.NewFromInt(int64(i)) + bidPrice := bid1Price.Sub(bidLayerSpread.Mul(fi)) + askPrice := ask1Price.Add(askLayerSpread.Mul(fi)) + + bidPrice = s.Market.TruncatePrice(bidPrice) + askPrice = s.Market.TruncatePrice(askPrice) + + bidPrices = append(bidPrices, bidPrice) + askPrices = append(askPrices, askPrice) + } + + availableBase := baseBal.Available + availableQuote := quoteBal.Available + + makerQuota := &bbgo.QuotaTransaction{} + makerQuota.QuoteAsset.Add(availableQuote) + makerQuota.BaseAsset.Add(availableBase) + + log.Infof("balances before liq orders: %s, %s", + baseBal.String(), + quoteBal.String()) + + if !s.Position.IsDust() { + if s.Position.IsLong() { + availableBase = availableBase.Sub(s.Position.Base) + availableBase = s.Market.RoundDownQuantityByPrecision(availableBase) + } else if s.Position.IsShort() { + posSizeInQuote := s.Position.Base.Mul(ticker.Sell) + availableQuote = availableQuote.Sub(posSizeInQuote) + } + } + + askX := availableBase.Float64() / askSum + bidX := availableQuote.Float64() / (bidSum * (fixedpoint.Sum(bidPrices).Float64())) + + askX = math.Trunc(askX*1e8) / 1e8 + bidX = math.Trunc(bidX*1e8) / 1e8 + + var liqOrders []types.SubmitOrder + for i := 0; i <= s.NumOfLiquidityLayers; i++ { + bidQuantity := fixedpoint.NewFromFloat(s.liquidityScale.Call(float64(i)) * bidX) + askQuantity := fixedpoint.NewFromFloat(s.liquidityScale.Call(float64(i)) * askX) + bidPrice := bidPrices[i] + askPrice := askPrices[i] + + log.Infof("liqudity layer #%d %f/%f = %f/%f", i, askPrice.Float64(), bidPrice.Float64(), askQuantity.Float64(), bidQuantity.Float64()) + + placeBuy := true + placeSell := true + averageCost := s.Position.AverageCost + // when long position, do not place sell orders below the average cost + if !s.Position.IsDust() { + if s.Position.IsLong() && askPrice.Compare(averageCost) < 0 { + placeSell = false + } + + if s.Position.IsShort() && bidPrice.Compare(averageCost) > 0 { + placeBuy = false + } + } + + quoteQuantity := bidQuantity.Mul(bidPrice) + + if s.Market.IsDustQuantity(bidQuantity, bidPrice) || !makerQuota.QuoteAsset.Lock(quoteQuantity) { + placeBuy = false + } + + if s.Market.IsDustQuantity(askQuantity, askPrice) || !makerQuota.BaseAsset.Lock(askQuantity) { + placeSell = false + } + + if placeBuy { + liqOrders = append(liqOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Side: types.SideTypeBuy, + Type: types.OrderTypeLimitMaker, + Quantity: bidQuantity, + Price: bidPrice, + Market: s.Market, + TimeInForce: types.TimeInForceGTC, + }) + } + + if placeSell { + liqOrders = append(liqOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Side: types.SideTypeSell, + Type: types.OrderTypeLimitMaker, + Quantity: askQuantity, + Price: askPrice, + Market: s.Market, + TimeInForce: types.TimeInForceGTC, + }) + } + } + + makerQuota.Commit() + + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, liqOrders...) + if logErr(err, "unable to place liquidity orders") { + return + } + + s.liquidityOrderBook.Add(createdOrders...) + log.Infof("%d liq orders are placed successfully", len(liqOrders)) +} + +func profitProtectedPrice( + side types.SideType, averageCost, price, feeRate, minProfit fixedpoint.Value, +) fixedpoint.Value { + switch side { + case types.SideTypeSell: + minProfitPrice := averageCost.Add( + averageCost.Mul(feeRate.Add(minProfit))) + return fixedpoint.Max(minProfitPrice, price) + + case types.SideTypeBuy: + minProfitPrice := averageCost.Sub( + averageCost.Mul(feeRate.Add(minProfit))) + return fixedpoint.Min(minProfitPrice, price) + + } + return price +} + +func logErr(err error, msgAndArgs ...interface{}) bool { + if err == nil { + return false + } + + if len(msgAndArgs) == 0 { + log.WithError(err).Error(err.Error()) + } else if len(msgAndArgs) == 1 { + msg := msgAndArgs[0].(string) + log.WithError(err).Error(msg) + } else if len(msgAndArgs) > 1 { + msg := msgAndArgs[0].(string) + log.WithError(err).Errorf(msg, msgAndArgs[1:]...) + } + + return true +} + +func preloadKLines( + inc *KLineStream, session *bbgo.ExchangeSession, symbol string, interval types.Interval, +) { + if store, ok := session.MarketDataStore(symbol); ok { + if kLinesData, ok := store.KLinesOfInterval(interval); ok { + for _, k := range *kLinesData { + inc.EmitUpdate(k) + } + } + } +} From 533907894e954c8c42a2b39486e93b596f025283 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 8 Nov 2023 15:37:12 +0800 Subject: [PATCH 25/33] liquiditymaker: implement order generator --- pkg/strategy/liquiditymaker/generator.go | 95 +++++++++++++++ pkg/strategy/liquiditymaker/generator_test.go | 114 ++++++++++++++++++ pkg/strategy/liquiditymaker/strategy.go | 3 + pkg/testing/testhelper/assert_priceside.go | 58 +++++++++ pkg/testing/testhelper/number.go | 18 +++ 5 files changed, 288 insertions(+) create mode 100644 pkg/strategy/liquiditymaker/generator.go create mode 100644 pkg/strategy/liquiditymaker/generator_test.go create mode 100644 pkg/testing/testhelper/assert_priceside.go create mode 100644 pkg/testing/testhelper/number.go diff --git a/pkg/strategy/liquiditymaker/generator.go b/pkg/strategy/liquiditymaker/generator.go new file mode 100644 index 000000000..2b19b3f48 --- /dev/null +++ b/pkg/strategy/liquiditymaker/generator.go @@ -0,0 +1,95 @@ +package liquiditymaker + +import ( + log "github.com/sirupsen/logrus" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +// input: liquidityOrderGenerator( +// +// totalLiquidityAmount, +// startPrice, +// endPrice, +// numLayers, +// quantityScale) +// +// when side == sell +// +// priceAsk1 * scale(1) * f = amount1 +// priceAsk2 * scale(2) * f = amount2 +// priceAsk3 * scale(3) * f = amount3 +// +// totalLiquidityAmount = priceAsk1 * scale(1) * f + priceAsk2 * scale(2) * f + priceAsk3 * scale(3) * f + .... +// totalLiquidityAmount = f * (priceAsk1 * scale(1) + priceAsk2 * scale(2) + priceAsk3 * scale(3) + ....) +// +// when side == buy +// +// priceBid1 * scale(1) * f = amount1 +type LiquidityOrderGenerator struct { + Symbol string + Market types.Market + + logger log.FieldLogger +} + +func (g *LiquidityOrderGenerator) Generate( + side types.SideType, totalAmount, startPrice, endPrice fixedpoint.Value, numLayers int, scale bbgo.Scale, +) (orders []types.SubmitOrder) { + + if g.logger == nil { + logger := log.New() + logger.SetLevel(log.ErrorLevel) + g.logger = logger + } + + layerSpread := endPrice.Sub(startPrice).Div(fixedpoint.NewFromInt(int64(numLayers - 1))) + switch side { + case types.SideTypeSell: + if layerSpread.Compare(g.Market.TickSize) < 0 { + layerSpread = g.Market.TickSize + } + + case types.SideTypeBuy: + if layerSpread.Compare(g.Market.TickSize.Neg()) > 0 { + layerSpread = g.Market.TickSize.Neg() + } + } + + quantityBase := 0.0 + var layerPrices []fixedpoint.Value + var layerScales []float64 + for i := 0; i < numLayers; i++ { + fi := fixedpoint.NewFromInt(int64(i)) + layerPrice := g.Market.TruncatePrice(startPrice.Add(layerSpread.Mul(fi))) + layerPrices = append(layerPrices, layerPrice) + + layerScale := scale.Call(float64(i + 1)) + layerScales = append(layerScales, layerScale) + + quantityBase += layerPrice.Float64() * layerScale + } + + factor := totalAmount.Float64() / quantityBase + + g.logger.Infof("liquidity amount base: %f, factor: %f", quantityBase, factor) + + for i := 0; i < numLayers; i++ { + price := layerPrices[i] + s := layerScales[i] + + quantity := factor * s + orders = append(orders, types.SubmitOrder{ + Symbol: g.Symbol, + Price: price, + Type: types.OrderTypeLimitMaker, + Quantity: g.Market.TruncateQuantity(fixedpoint.NewFromFloat(quantity)), + Side: side, + Market: g.Market, + }) + } + + return orders +} diff --git a/pkg/strategy/liquiditymaker/generator_test.go b/pkg/strategy/liquiditymaker/generator_test.go new file mode 100644 index 000000000..d56700f6e --- /dev/null +++ b/pkg/strategy/liquiditymaker/generator_test.go @@ -0,0 +1,114 @@ +//go:build !dnum + +package liquiditymaker + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + . "github.com/c9s/bbgo/pkg/testing/testhelper" + "github.com/c9s/bbgo/pkg/types" +) + +func newTestMarket() types.Market { + return types.Market{ + BaseCurrency: "XML", + QuoteCurrency: "USDT", + TickSize: Number(0.0001), + StepSize: Number(0.01), + PricePrecision: 4, + VolumePrecision: 8, + MinNotional: Number(8.0), + MinQuantity: Number(40.0), + } +} + +func TestLiquidityOrderGenerator(t *testing.T) { + g := &LiquidityOrderGenerator{ + Symbol: "XMLUSDT", + Market: newTestMarket(), + } + + scale := &bbgo.ExponentialScale{ + Domain: [2]float64{1.0, 30.0}, + Range: [2]float64{1.0, 4.0}, + } + + err := scale.Solve() + assert.NoError(t, err) + assert.InDelta(t, 1.0, scale.Call(1.0), 0.00001) + assert.InDelta(t, 4.0, scale.Call(30.0), 0.00001) + + totalAmount := Number(200_000.0) + + t.Run("ask orders", func(t *testing.T) { + orders := g.Generate(types.SideTypeSell, totalAmount, Number(2.0), Number(2.04), 30, scale) + assert.Len(t, orders, 30) + + totalQuoteQuantity := fixedpoint.NewFromInt(0) + for _, o := range orders { + totalQuoteQuantity = totalQuoteQuantity.Add(o.Quantity.Mul(o.Price)) + } + assert.InDelta(t, totalAmount.Float64(), totalQuoteQuantity.Float64(), 1.0) + + AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ + {Side: types.SideTypeSell, Price: Number("2.0000"), Quantity: Number("1513.40")}, + {Side: types.SideTypeSell, Price: Number("2.0013"), Quantity: Number("1587.50")}, + {Side: types.SideTypeSell, Price: Number("2.0027"), Quantity: Number("1665.23")}, + {Side: types.SideTypeSell, Price: Number("2.0041"), Quantity: Number("1746.77")}, + {Side: types.SideTypeSell, Price: Number("2.0055"), Quantity: Number("1832.30")}, + {Side: types.SideTypeSell, Price: Number("2.0068"), Quantity: Number("1922.02")}, + {Side: types.SideTypeSell, Price: Number("2.0082"), Quantity: Number("2016.13")}, + {Side: types.SideTypeSell, Price: Number("2.0096"), Quantity: Number("2114.85")}, + {Side: types.SideTypeSell, Price: Number("2.0110"), Quantity: Number("2218.40")}, + {Side: types.SideTypeSell, Price: Number("2.0124"), Quantity: Number("2327.02")}, + {Side: types.SideTypeSell, Price: Number("2.0137"), Quantity: Number("2440.96")}, + {Side: types.SideTypeSell, Price: Number("2.0151"), Quantity: Number("2560.48")}, + {Side: types.SideTypeSell, Price: Number("2.0165"), Quantity: Number("2685.86")}, + {Side: types.SideTypeSell, Price: Number("2.0179"), Quantity: Number("2817.37")}, + {Side: types.SideTypeSell, Price: Number("2.0193"), Quantity: Number("2955.32")}, + }, orders[0:15]) + + AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ + {Side: types.SideTypeSell, Price: Number("2.0386"), Quantity: Number("5771.04")}, + {Side: types.SideTypeSell, Price: Number("2.0399"), Quantity: Number("6053.62")}, + }, orders[28:30]) + }) + + t.Run("bid orders", func(t *testing.T) { + orders := g.Generate(types.SideTypeBuy, totalAmount, Number(2.0), Number(1.96), 30, scale) + assert.Len(t, orders, 30) + + totalQuoteQuantity := fixedpoint.NewFromInt(0) + for _, o := range orders { + totalQuoteQuantity = totalQuoteQuantity.Add(o.Quantity.Mul(o.Price)) + } + assert.InDelta(t, totalAmount.Float64(), totalQuoteQuantity.Float64(), 1.0) + + AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ + {Side: types.SideTypeBuy, Price: Number("2.0000"), Quantity: Number("1551.37")}, + {Side: types.SideTypeBuy, Price: Number("1.9986"), Quantity: Number("1627.33")}, + {Side: types.SideTypeBuy, Price: Number("1.9972"), Quantity: Number("1707.01")}, + {Side: types.SideTypeBuy, Price: Number("1.9958"), Quantity: Number("1790.59")}, + {Side: types.SideTypeBuy, Price: Number("1.9944"), Quantity: Number("1878.27")}, + {Side: types.SideTypeBuy, Price: Number("1.9931"), Quantity: Number("1970.24")}, + {Side: types.SideTypeBuy, Price: Number("1.9917"), Quantity: Number("2066.71")}, + {Side: types.SideTypeBuy, Price: Number("1.9903"), Quantity: Number("2167.91")}, + {Side: types.SideTypeBuy, Price: Number("1.9889"), Quantity: Number("2274.06")}, + {Side: types.SideTypeBuy, Price: Number("1.9875"), Quantity: Number("2385.40")}, + {Side: types.SideTypeBuy, Price: Number("1.9862"), Quantity: Number("2502.20")}, + {Side: types.SideTypeBuy, Price: Number("1.9848"), Quantity: Number("2624.72")}, + {Side: types.SideTypeBuy, Price: Number("1.9834"), Quantity: Number("2753.24")}, + {Side: types.SideTypeBuy, Price: Number("1.9820"), Quantity: Number("2888.05")}, + {Side: types.SideTypeBuy, Price: Number("1.9806"), Quantity: Number("3029.46")}, + }, orders[0:15]) + + AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ + {Side: types.SideTypeBuy, Price: Number("1.9613"), Quantity: Number("5915.83")}, + {Side: types.SideTypeBuy, Price: Number("1.9600"), Quantity: Number("6205.49")}, + }, orders[28:30]) + }) +} diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go index fcd8f17de..6dcb8ece0 100644 --- a/pkg/strategy/liquiditymaker/strategy.go +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -50,6 +50,9 @@ type Strategy struct { LiquiditySkew fixedpoint.Value `json:"liquiditySkew"` LiquidityPriceRange fixedpoint.Value `json:"liquidityPriceRange"` + AskLiquidityAmount fixedpoint.Value `json:"askLiquidityAmount"` + BidLiquidityAmount fixedpoint.Value `json:"bidLiquidityAmount"` + Spread fixedpoint.Value `json:"spread"` MaxPrice fixedpoint.Value `json:"maxPrice"` MinPrice fixedpoint.Value `json:"minPrice"` diff --git a/pkg/testing/testhelper/assert_priceside.go b/pkg/testing/testhelper/assert_priceside.go new file mode 100644 index 000000000..7c45cdd9d --- /dev/null +++ b/pkg/testing/testhelper/assert_priceside.go @@ -0,0 +1,58 @@ +package testhelper + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +type PriceSideAssert struct { + Price fixedpoint.Value + Side types.SideType +} + +// AssertOrdersPriceSide asserts the orders with the given price and side (slice) +func AssertOrdersPriceSide(t *testing.T, asserts []PriceSideAssert, orders []types.SubmitOrder) { + for i, a := range asserts { + assert.Equalf(t, a.Price, orders[i].Price, "order #%d price should be %f", i+1, a.Price.Float64()) + assert.Equalf(t, a.Side, orders[i].Side, "order at price %f should be %s", a.Price.Float64(), a.Side) + } +} + +type PriceSideQuantityAssert struct { + Price fixedpoint.Value + Side types.SideType + Quantity fixedpoint.Value +} + +// AssertOrdersPriceSide asserts the orders with the given price and side (slice) +func AssertOrdersPriceSideQuantity( + t *testing.T, asserts []PriceSideQuantityAssert, orders []types.SubmitOrder, +) { + assert.Equalf(t, len(orders), len(asserts), "expecting %d orders", len(asserts)) + + var assertPrices, orderPrices fixedpoint.Slice + var assertPricesFloat, orderPricesFloat []float64 + for _, a := range asserts { + assertPrices = append(assertPrices, a.Price) + assertPricesFloat = append(assertPricesFloat, a.Price.Float64()) + } + + for _, o := range orders { + orderPrices = append(orderPrices, o.Price) + orderPricesFloat = append(orderPricesFloat, o.Price.Float64()) + } + + if !assert.Equalf(t, assertPricesFloat, orderPricesFloat, "assert prices") { + return + } + + for i, a := range asserts { + assert.Equalf(t, a.Price.Float64(), orders[i].Price.Float64(), "order #%d price should be %f", i+1, a.Price.Float64()) + assert.Equalf(t, a.Quantity.Float64(), orders[i].Quantity.Float64(), "order #%d quantity should be %f", i+1, a.Quantity.Float64()) + assert.Equalf(t, a.Side, orders[i].Side, "order at price %f should be %s", a.Price.Float64(), a.Side) + } +} diff --git a/pkg/testing/testhelper/number.go b/pkg/testing/testhelper/number.go new file mode 100644 index 000000000..e57659a01 --- /dev/null +++ b/pkg/testing/testhelper/number.go @@ -0,0 +1,18 @@ +package testhelper + +import "github.com/c9s/bbgo/pkg/fixedpoint" + +func Number(a interface{}) fixedpoint.Value { + switch v := a.(type) { + case string: + return fixedpoint.MustNewFromString(v) + case int: + return fixedpoint.NewFromInt(int64(v)) + case int64: + return fixedpoint.NewFromInt(int64(v)) + case float64: + return fixedpoint.NewFromFloat(v) + } + + return fixedpoint.Zero +} From cc5c033af76c2dbd341140b8328c88c3f95bed6c Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 8 Nov 2023 17:54:01 +0800 Subject: [PATCH 26/33] liquiditymaker: use order generator --- config/liquiditymaker.yaml | 54 +++++ pkg/cmd/strategy/builtin.go | 1 + pkg/strategy/liquiditymaker/generator_test.go | 70 +++---- pkg/strategy/liquiditymaker/strategy.go | 185 +++++------------- 4 files changed, 142 insertions(+), 168 deletions(-) create mode 100644 config/liquiditymaker.yaml diff --git a/config/liquiditymaker.yaml b/config/liquiditymaker.yaml new file mode 100644 index 000000000..85288d5b9 --- /dev/null +++ b/config/liquiditymaker.yaml @@ -0,0 +1,54 @@ +sessions: + max: + exchange: max + envVarPrefix: max + makerFeeRate: 0% + takerFeeRate: 0.025% + +#services: +# googleSpreadSheet: +# jsonTokenFile: ".credentials/google-cloud/service-account-json-token.json" +# spreadSheetId: "YOUR_SPREADSHEET_ID" + +exchangeStrategies: +- on: max + liquiditymaker: + symbol: &symbol USDTTWD + + ## adjustmentUpdateInterval is the interval for adjusting position + adjustmentUpdateInterval: 1m + + ## liquidityUpdateInterval is the interval for updating liquidity orders + liquidityUpdateInterval: 1h + + numOfLiquidityLayers: 30 + askLiquidityAmount: 20_000.0 + bidLiquidityAmount: 20_000.0 + liquidityPriceRange: 2% + useLastTradePrice: true + spread: 1.1% + + liquidityScale: + exp: + domain: [1, 30] + range: [1, 4] + + ## maxExposure controls how much balance should be used for placing the maker orders + maxExposure: 200_000 + minProfit: 0.01% + + +backtest: + sessions: + - max + startTime: "2023-05-20" + endTime: "2023-06-01" + symbols: + - *symbol + account: + max: + makerFeeRate: 0.0% + takerFeeRate: 0.025% + balances: + USDT: 5000 + TWD: 150_000 diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index d868e926a..867c72dc2 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -25,6 +25,7 @@ import ( _ "github.com/c9s/bbgo/pkg/strategy/irr" _ "github.com/c9s/bbgo/pkg/strategy/kline" _ "github.com/c9s/bbgo/pkg/strategy/linregmaker" + _ "github.com/c9s/bbgo/pkg/strategy/liquiditymaker" _ "github.com/c9s/bbgo/pkg/strategy/marketcap" _ "github.com/c9s/bbgo/pkg/strategy/pivotshort" _ "github.com/c9s/bbgo/pkg/strategy/pricealert" diff --git a/pkg/strategy/liquiditymaker/generator_test.go b/pkg/strategy/liquiditymaker/generator_test.go index d56700f6e..995f33bd7 100644 --- a/pkg/strategy/liquiditymaker/generator_test.go +++ b/pkg/strategy/liquiditymaker/generator_test.go @@ -42,7 +42,7 @@ func TestLiquidityOrderGenerator(t *testing.T) { assert.InDelta(t, 1.0, scale.Call(1.0), 0.00001) assert.InDelta(t, 4.0, scale.Call(30.0), 0.00001) - totalAmount := Number(200_000.0) + totalAmount := Number(20_000.0) t.Run("ask orders", func(t *testing.T) { orders := g.Generate(types.SideTypeSell, totalAmount, Number(2.0), Number(2.04), 30, scale) @@ -55,26 +55,26 @@ func TestLiquidityOrderGenerator(t *testing.T) { assert.InDelta(t, totalAmount.Float64(), totalQuoteQuantity.Float64(), 1.0) AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ - {Side: types.SideTypeSell, Price: Number("2.0000"), Quantity: Number("1513.40")}, - {Side: types.SideTypeSell, Price: Number("2.0013"), Quantity: Number("1587.50")}, - {Side: types.SideTypeSell, Price: Number("2.0027"), Quantity: Number("1665.23")}, - {Side: types.SideTypeSell, Price: Number("2.0041"), Quantity: Number("1746.77")}, - {Side: types.SideTypeSell, Price: Number("2.0055"), Quantity: Number("1832.30")}, - {Side: types.SideTypeSell, Price: Number("2.0068"), Quantity: Number("1922.02")}, - {Side: types.SideTypeSell, Price: Number("2.0082"), Quantity: Number("2016.13")}, - {Side: types.SideTypeSell, Price: Number("2.0096"), Quantity: Number("2114.85")}, - {Side: types.SideTypeSell, Price: Number("2.0110"), Quantity: Number("2218.40")}, - {Side: types.SideTypeSell, Price: Number("2.0124"), Quantity: Number("2327.02")}, - {Side: types.SideTypeSell, Price: Number("2.0137"), Quantity: Number("2440.96")}, - {Side: types.SideTypeSell, Price: Number("2.0151"), Quantity: Number("2560.48")}, - {Side: types.SideTypeSell, Price: Number("2.0165"), Quantity: Number("2685.86")}, - {Side: types.SideTypeSell, Price: Number("2.0179"), Quantity: Number("2817.37")}, - {Side: types.SideTypeSell, Price: Number("2.0193"), Quantity: Number("2955.32")}, + {Side: types.SideTypeSell, Price: Number("2.0000"), Quantity: Number("151.34")}, + {Side: types.SideTypeSell, Price: Number("2.0013"), Quantity: Number("158.75")}, + {Side: types.SideTypeSell, Price: Number("2.0027"), Quantity: Number("166.52")}, + {Side: types.SideTypeSell, Price: Number("2.0041"), Quantity: Number("174.67")}, + {Side: types.SideTypeSell, Price: Number("2.0055"), Quantity: Number("183.23")}, + {Side: types.SideTypeSell, Price: Number("2.0068"), Quantity: Number("192.20")}, + {Side: types.SideTypeSell, Price: Number("2.0082"), Quantity: Number("201.61")}, + {Side: types.SideTypeSell, Price: Number("2.0096"), Quantity: Number("211.48")}, + {Side: types.SideTypeSell, Price: Number("2.0110"), Quantity: Number("221.84")}, + {Side: types.SideTypeSell, Price: Number("2.0124"), Quantity: Number("232.70")}, + {Side: types.SideTypeSell, Price: Number("2.0137"), Quantity: Number("244.09")}, + {Side: types.SideTypeSell, Price: Number("2.0151"), Quantity: Number("256.04")}, + {Side: types.SideTypeSell, Price: Number("2.0165"), Quantity: Number("268.58")}, + {Side: types.SideTypeSell, Price: Number("2.0179"), Quantity: Number("281.73")}, + {Side: types.SideTypeSell, Price: Number("2.0193"), Quantity: Number("295.53")}, }, orders[0:15]) AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ - {Side: types.SideTypeSell, Price: Number("2.0386"), Quantity: Number("5771.04")}, - {Side: types.SideTypeSell, Price: Number("2.0399"), Quantity: Number("6053.62")}, + {Side: types.SideTypeSell, Price: Number("2.0386"), Quantity: Number("577.10")}, + {Side: types.SideTypeSell, Price: Number("2.0399"), Quantity: Number("605.36")}, }, orders[28:30]) }) @@ -89,26 +89,26 @@ func TestLiquidityOrderGenerator(t *testing.T) { assert.InDelta(t, totalAmount.Float64(), totalQuoteQuantity.Float64(), 1.0) AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ - {Side: types.SideTypeBuy, Price: Number("2.0000"), Quantity: Number("1551.37")}, - {Side: types.SideTypeBuy, Price: Number("1.9986"), Quantity: Number("1627.33")}, - {Side: types.SideTypeBuy, Price: Number("1.9972"), Quantity: Number("1707.01")}, - {Side: types.SideTypeBuy, Price: Number("1.9958"), Quantity: Number("1790.59")}, - {Side: types.SideTypeBuy, Price: Number("1.9944"), Quantity: Number("1878.27")}, - {Side: types.SideTypeBuy, Price: Number("1.9931"), Quantity: Number("1970.24")}, - {Side: types.SideTypeBuy, Price: Number("1.9917"), Quantity: Number("2066.71")}, - {Side: types.SideTypeBuy, Price: Number("1.9903"), Quantity: Number("2167.91")}, - {Side: types.SideTypeBuy, Price: Number("1.9889"), Quantity: Number("2274.06")}, - {Side: types.SideTypeBuy, Price: Number("1.9875"), Quantity: Number("2385.40")}, - {Side: types.SideTypeBuy, Price: Number("1.9862"), Quantity: Number("2502.20")}, - {Side: types.SideTypeBuy, Price: Number("1.9848"), Quantity: Number("2624.72")}, - {Side: types.SideTypeBuy, Price: Number("1.9834"), Quantity: Number("2753.24")}, - {Side: types.SideTypeBuy, Price: Number("1.9820"), Quantity: Number("2888.05")}, - {Side: types.SideTypeBuy, Price: Number("1.9806"), Quantity: Number("3029.46")}, + {Side: types.SideTypeBuy, Price: Number("2.0000"), Quantity: Number("155.13")}, + {Side: types.SideTypeBuy, Price: Number("1.9986"), Quantity: Number("162.73")}, + {Side: types.SideTypeBuy, Price: Number("1.9972"), Quantity: Number("170.70")}, + {Side: types.SideTypeBuy, Price: Number("1.9958"), Quantity: Number("179.05")}, + {Side: types.SideTypeBuy, Price: Number("1.9944"), Quantity: Number("187.82")}, + {Side: types.SideTypeBuy, Price: Number("1.9931"), Quantity: Number("197.02")}, + {Side: types.SideTypeBuy, Price: Number("1.9917"), Quantity: Number("206.67")}, + {Side: types.SideTypeBuy, Price: Number("1.9903"), Quantity: Number("216.79")}, + {Side: types.SideTypeBuy, Price: Number("1.9889"), Quantity: Number("227.40")}, + {Side: types.SideTypeBuy, Price: Number("1.9875"), Quantity: Number("238.54")}, + {Side: types.SideTypeBuy, Price: Number("1.9862"), Quantity: Number("250.22")}, + {Side: types.SideTypeBuy, Price: Number("1.9848"), Quantity: Number("262.47")}, + {Side: types.SideTypeBuy, Price: Number("1.9834"), Quantity: Number("275.32")}, + {Side: types.SideTypeBuy, Price: Number("1.9820"), Quantity: Number("288.80")}, + {Side: types.SideTypeBuy, Price: Number("1.9806"), Quantity: Number("302.94")}, }, orders[0:15]) AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ - {Side: types.SideTypeBuy, Price: Number("1.9613"), Quantity: Number("5915.83")}, - {Side: types.SideTypeBuy, Price: Number("1.9600"), Quantity: Number("6205.49")}, + {Side: types.SideTypeBuy, Price: Number("1.9613"), Quantity: Number("591.58")}, + {Side: types.SideTypeBuy, Price: Number("1.9600"), Quantity: Number("620.54")}, }, orders[28:30]) }) } diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go index 6dcb8ece0..07a6517e7 100644 --- a/pkg/strategy/liquiditymaker/strategy.go +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -3,7 +3,6 @@ package liquiditymaker import ( "context" "fmt" - "math" "sync" log "github.com/sirupsen/logrus" @@ -44,18 +43,16 @@ type Strategy struct { AdjustmentUpdateInterval types.Interval `json:"adjustmentUpdateInterval"` - NumOfLiquidityLayers int `json:"numOfLiquidityLayers"` - LiquiditySlideRule *bbgo.SlideRule `json:"liquidityScale"` - LiquidityLayerTickSize fixedpoint.Value `json:"liquidityLayerTickSize"` - LiquiditySkew fixedpoint.Value `json:"liquiditySkew"` - LiquidityPriceRange fixedpoint.Value `json:"liquidityPriceRange"` + NumOfLiquidityLayers int `json:"numOfLiquidityLayers"` + LiquiditySlideRule *bbgo.SlideRule `json:"liquidityScale"` + LiquidityPriceRange fixedpoint.Value `json:"liquidityPriceRange"` + AskLiquidityAmount fixedpoint.Value `json:"askLiquidityAmount"` + BidLiquidityAmount fixedpoint.Value `json:"bidLiquidityAmount"` - AskLiquidityAmount fixedpoint.Value `json:"askLiquidityAmount"` - BidLiquidityAmount fixedpoint.Value `json:"bidLiquidityAmount"` - - Spread fixedpoint.Value `json:"spread"` - MaxPrice fixedpoint.Value `json:"maxPrice"` - MinPrice fixedpoint.Value `json:"minPrice"` + UseLastTradePrice bool `json:"useLastTradePrice"` + Spread fixedpoint.Value `json:"spread"` + MaxPrice fixedpoint.Value `json:"maxPrice"` + MinPrice fixedpoint.Value `json:"minPrice"` MaxExposure fixedpoint.Value `json:"maxExposure"` @@ -65,6 +62,8 @@ type Strategy struct { book *types.StreamOrderBook liquidityScale bbgo.Scale + + orderGenerator *LiquidityOrderGenerator } func (s *Strategy) ID() string { @@ -85,6 +84,11 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) + s.orderGenerator = &LiquidityOrderGenerator{ + Symbol: s.Symbol, + Market: s.Market, + } + s.book = types.NewStreamBook(s.Symbol) s.book.BindStream(session.MarketDataStream) @@ -209,6 +213,11 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context) { } func (s *Strategy) placeLiquidityOrders(ctx context.Context) { + err := s.liquidityOrderBook.GracefulCancel(ctx, s.Session.Exchange) + if logErr(err, "unable to cancel orders") { + return + } + ticker, err := s.Session.Exchange.QueryTicker(ctx, s.Symbol) if logErr(err, "unable to query ticker") { return @@ -219,11 +228,14 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { return } - err = s.liquidityOrderBook.GracefulCancel(ctx, s.Session.Exchange) - if logErr(err, "unable to cancel orders") { + if _, err := s.Session.UpdateAccount(ctx); err != nil { + logErr(err, "unable to update account") return } + baseBal, _ := s.Session.Account.Balance(s.Market.BaseCurrency) + quoteBal, _ := s.Session.Account.Balance(s.Market.QuoteCurrency) + if ticker.Buy.IsZero() && ticker.Sell.IsZero() { ticker.Sell = ticker.Last.Add(s.Market.TickSize) ticker.Buy = ticker.Last.Sub(s.Market.TickSize) @@ -233,78 +245,32 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { ticker.Sell = ticker.Buy.Add(s.Market.TickSize) } - if _, err := s.Session.UpdateAccount(ctx); err != nil { - logErr(err, "unable to update account") - return - } - - baseBal, _ := s.Session.Account.Balance(s.Market.BaseCurrency) - quoteBal, _ := s.Session.Account.Balance(s.Market.QuoteCurrency) + log.Infof("ticker: %+v", ticker) lastTradedPrice := ticker.Last midPrice := ticker.Sell.Add(ticker.Buy).Div(fixedpoint.Two) currentSpread := ticker.Sell.Sub(ticker.Buy) - tickSize := fixedpoint.Max(s.LiquidityLayerTickSize, s.Market.TickSize) sideSpread := s.Spread.Div(fixedpoint.Two) - log.Infof("current: spread: %f lastTradedPrice: %f midPrice: %f", currentSpread.Float64(), lastTradedPrice.Float64(), midPrice.Float64()) + if s.UseLastTradePrice { + midPrice = lastTradedPrice + } + + log.Infof("current spread: %f lastTradedPrice: %f midPrice: %f", currentSpread.Float64(), lastTradedPrice.Float64(), midPrice.Float64()) ask1Price := midPrice.Mul(fixedpoint.One.Add(sideSpread)) bid1Price := midPrice.Mul(fixedpoint.One.Sub(sideSpread)) askLastPrice := midPrice.Mul(fixedpoint.One.Add(s.LiquidityPriceRange)) bidLastPrice := midPrice.Mul(fixedpoint.One.Sub(s.LiquidityPriceRange)) - log.Infof("wanted side spread: %f askRange: %f ~ %f bidRange: %f ~ %f", sideSpread.Float64(), + log.Infof("wanted side spread: %f askRange: %f ~ %f bidRange: %f ~ %f", + sideSpread.Float64(), ask1Price.Float64(), askLastPrice.Float64(), bid1Price.Float64(), bidLastPrice.Float64()) - askLayerSpread := askLastPrice.Sub(ask1Price).Div(fixedpoint.NewFromInt(int64(s.NumOfLiquidityLayers))) - bidLayerSpread := bid1Price.Sub(bidLastPrice).Div(fixedpoint.NewFromInt(int64(s.NumOfLiquidityLayers))) - - if askLayerSpread.Compare(tickSize) < 0 { - askLayerSpread = tickSize - } - - if bidLayerSpread.Compare(tickSize) < 0 { - bidLayerSpread = tickSize - } - - sum := s.liquidityScale.Sum(1.0) - askSum := sum - bidSum := sum - log.Infof("liquidity sum: %f / %f", askSum, bidSum) - - skew := s.LiquiditySkew.Float64() - useSkew := !s.LiquiditySkew.IsZero() - if useSkew { - askSum = sum / skew - bidSum = sum * skew - log.Infof("adjusted liqudity skew: %f / %f", askSum, bidSum) - } - - var bidPrices []fixedpoint.Value - var askPrices []fixedpoint.Value - - // calculate and collect prices - for i := 0; i <= s.NumOfLiquidityLayers; i++ { - fi := fixedpoint.NewFromInt(int64(i)) - bidPrice := bid1Price.Sub(bidLayerSpread.Mul(fi)) - askPrice := ask1Price.Add(askLayerSpread.Mul(fi)) - - bidPrice = s.Market.TruncatePrice(bidPrice) - askPrice = s.Market.TruncatePrice(askPrice) - - bidPrices = append(bidPrices, bidPrice) - askPrices = append(askPrices, askPrice) - } - availableBase := baseBal.Available availableQuote := quoteBal.Available - makerQuota := &bbgo.QuotaTransaction{} - makerQuota.QuoteAsset.Add(availableQuote) - makerQuota.BaseAsset.Add(availableBase) - log.Infof("balances before liq orders: %s, %s", baseBal.String(), quoteBal.String()) @@ -319,79 +285,32 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { } } - askX := availableBase.Float64() / askSum - bidX := availableQuote.Float64() / (bidSum * (fixedpoint.Sum(bidPrices).Float64())) + bidOrders := s.orderGenerator.Generate(types.SideTypeBuy, + fixedpoint.Min(s.BidLiquidityAmount, quoteBal.Available), + bid1Price, + bidLastPrice, + s.NumOfLiquidityLayers, + s.liquidityScale) - askX = math.Trunc(askX*1e8) / 1e8 - bidX = math.Trunc(bidX*1e8) / 1e8 + askOrders := s.orderGenerator.Generate(types.SideTypeSell, + s.AskLiquidityAmount, + ask1Price, + askLastPrice, + s.NumOfLiquidityLayers, + s.liquidityScale) - var liqOrders []types.SubmitOrder - for i := 0; i <= s.NumOfLiquidityLayers; i++ { - bidQuantity := fixedpoint.NewFromFloat(s.liquidityScale.Call(float64(i)) * bidX) - askQuantity := fixedpoint.NewFromFloat(s.liquidityScale.Call(float64(i)) * askX) - bidPrice := bidPrices[i] - askPrice := askPrices[i] + orderForms := append(bidOrders, askOrders...) - log.Infof("liqudity layer #%d %f/%f = %f/%f", i, askPrice.Float64(), bidPrice.Float64(), askQuantity.Float64(), bidQuantity.Float64()) - - placeBuy := true - placeSell := true - averageCost := s.Position.AverageCost - // when long position, do not place sell orders below the average cost - if !s.Position.IsDust() { - if s.Position.IsLong() && askPrice.Compare(averageCost) < 0 { - placeSell = false - } - - if s.Position.IsShort() && bidPrice.Compare(averageCost) > 0 { - placeBuy = false - } - } - - quoteQuantity := bidQuantity.Mul(bidPrice) - - if s.Market.IsDustQuantity(bidQuantity, bidPrice) || !makerQuota.QuoteAsset.Lock(quoteQuantity) { - placeBuy = false - } - - if s.Market.IsDustQuantity(askQuantity, askPrice) || !makerQuota.BaseAsset.Lock(askQuantity) { - placeSell = false - } - - if placeBuy { - liqOrders = append(liqOrders, types.SubmitOrder{ - Symbol: s.Symbol, - Side: types.SideTypeBuy, - Type: types.OrderTypeLimitMaker, - Quantity: bidQuantity, - Price: bidPrice, - Market: s.Market, - TimeInForce: types.TimeInForceGTC, - }) - } - - if placeSell { - liqOrders = append(liqOrders, types.SubmitOrder{ - Symbol: s.Symbol, - Side: types.SideTypeSell, - Type: types.OrderTypeLimitMaker, - Quantity: askQuantity, - Price: askPrice, - Market: s.Market, - TimeInForce: types.TimeInForceGTC, - }) - } - } - - makerQuota.Commit() - - createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, liqOrders...) + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, orderForms...) if logErr(err, "unable to place liquidity orders") { return } s.liquidityOrderBook.Add(createdOrders...) - log.Infof("%d liq orders are placed successfully", len(liqOrders)) + log.Infof("%d liq orders are placed successfully", len(orderForms)) + for _, o := range createdOrders { + log.Infof("liq order: %+v", o) + } } func profitProtectedPrice( From 3563c0b98601066aeefb0300ff02d6955ef84805 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 8 Nov 2023 20:19:05 +0800 Subject: [PATCH 27/33] liquiditymaker: filterAskOrders by base balance --- pkg/strategy/liquiditymaker/generator.go | 1 + pkg/strategy/liquiditymaker/strategy.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkg/strategy/liquiditymaker/generator.go b/pkg/strategy/liquiditymaker/generator.go index 2b19b3f48..d21d79b83 100644 --- a/pkg/strategy/liquiditymaker/generator.go +++ b/pkg/strategy/liquiditymaker/generator.go @@ -24,6 +24,7 @@ import ( // // totalLiquidityAmount = priceAsk1 * scale(1) * f + priceAsk2 * scale(2) * f + priceAsk3 * scale(3) * f + .... // totalLiquidityAmount = f * (priceAsk1 * scale(1) + priceAsk2 * scale(2) + priceAsk3 * scale(3) + ....) +// f = totalLiquidityAmount / (priceAsk1 * scale(1) + priceAsk2 * scale(2) + priceAsk3 * scale(3) + ....) // // when side == buy // diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go index 07a6517e7..9d90e8fed 100644 --- a/pkg/strategy/liquiditymaker/strategy.go +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -299,6 +299,8 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { s.NumOfLiquidityLayers, s.liquidityScale) + askOrders = filterAskOrders(askOrders, baseBal.Available) + orderForms := append(bidOrders, askOrders...) createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, orderForms...) @@ -331,6 +333,20 @@ func profitProtectedPrice( return price } +func filterAskOrders(askOrders []types.SubmitOrder, available fixedpoint.Value) (out []types.SubmitOrder) { + usedBase := fixedpoint.Zero + for _, askOrder := range askOrders { + if usedBase.Add(askOrder.Quantity).Compare(available) > 0 { + return out + } + + usedBase = usedBase.Add(askOrder.Quantity) + out = append(out, askOrder) + } + + return out +} + func logErr(err error, msgAndArgs ...interface{}) bool { if err == nil { return false From 3d1de0ca058fd852740947621ff2f31472150aa0 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 9 Nov 2023 12:56:18 +0800 Subject: [PATCH 28/33] update command doc files --- doc/commands/bbgo.md | 2 +- doc/commands/bbgo_account.md | 2 +- doc/commands/bbgo_backtest.md | 2 +- doc/commands/bbgo_balances.md | 2 +- doc/commands/bbgo_build.md | 2 +- doc/commands/bbgo_cancel-order.md | 2 +- doc/commands/bbgo_deposits.md | 2 +- doc/commands/bbgo_execute-order.md | 2 +- doc/commands/bbgo_get-order.md | 2 +- doc/commands/bbgo_hoptimize.md | 2 +- doc/commands/bbgo_kline.md | 2 +- doc/commands/bbgo_list-orders.md | 2 +- doc/commands/bbgo_margin.md | 2 +- doc/commands/bbgo_margin_interests.md | 2 +- doc/commands/bbgo_margin_loans.md | 2 +- doc/commands/bbgo_margin_repays.md | 2 +- doc/commands/bbgo_market.md | 2 +- doc/commands/bbgo_optimize.md | 2 +- doc/commands/bbgo_orderbook.md | 2 +- doc/commands/bbgo_orderupdate.md | 2 +- doc/commands/bbgo_pnl.md | 2 +- doc/commands/bbgo_run.md | 2 +- doc/commands/bbgo_submit-order.md | 2 +- doc/commands/bbgo_sync.md | 2 +- doc/commands/bbgo_trades.md | 2 +- doc/commands/bbgo_tradeupdate.md | 2 +- doc/commands/bbgo_transfer-history.md | 2 +- doc/commands/bbgo_userdatastream.md | 2 +- doc/commands/bbgo_version.md | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/commands/bbgo.md b/doc/commands/bbgo.md index cf60746a8..605f188bc 100644 --- a/doc/commands/bbgo.md +++ b/doc/commands/bbgo.md @@ -58,4 +58,4 @@ bbgo [flags] * [bbgo userdatastream](bbgo_userdatastream.md) - Listen to session events (orderUpdate, tradeUpdate, balanceUpdate, balanceSnapshot) * [bbgo version](bbgo_version.md) - show version name -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_account.md b/doc/commands/bbgo_account.md index 973650f24..ddd67acf4 100644 --- a/doc/commands/bbgo_account.md +++ b/doc/commands/bbgo_account.md @@ -41,4 +41,4 @@ bbgo account [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_backtest.md b/doc/commands/bbgo_backtest.md index f8c477e0b..4c89aa619 100644 --- a/doc/commands/bbgo_backtest.md +++ b/doc/commands/bbgo_backtest.md @@ -50,4 +50,4 @@ bbgo backtest [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_balances.md b/doc/commands/bbgo_balances.md index 634db2863..014c4d5a8 100644 --- a/doc/commands/bbgo_balances.md +++ b/doc/commands/bbgo_balances.md @@ -40,4 +40,4 @@ bbgo balances [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_build.md b/doc/commands/bbgo_build.md index 77010d6d8..8d679a454 100644 --- a/doc/commands/bbgo_build.md +++ b/doc/commands/bbgo_build.md @@ -39,4 +39,4 @@ bbgo build [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_cancel-order.md b/doc/commands/bbgo_cancel-order.md index 2d0fcdfdc..956e07afb 100644 --- a/doc/commands/bbgo_cancel-order.md +++ b/doc/commands/bbgo_cancel-order.md @@ -49,4 +49,4 @@ bbgo cancel-order [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_deposits.md b/doc/commands/bbgo_deposits.md index a59a6bb68..42754dcf1 100644 --- a/doc/commands/bbgo_deposits.md +++ b/doc/commands/bbgo_deposits.md @@ -41,4 +41,4 @@ bbgo deposits [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_execute-order.md b/doc/commands/bbgo_execute-order.md index 10b5a5942..05d285188 100644 --- a/doc/commands/bbgo_execute-order.md +++ b/doc/commands/bbgo_execute-order.md @@ -48,4 +48,4 @@ bbgo execute-order --session SESSION --symbol SYMBOL --side SIDE --target-quanti * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_get-order.md b/doc/commands/bbgo_get-order.md index b18f69fc6..5fe462deb 100644 --- a/doc/commands/bbgo_get-order.md +++ b/doc/commands/bbgo_get-order.md @@ -42,4 +42,4 @@ bbgo get-order --session SESSION --order-id ORDER_ID [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_hoptimize.md b/doc/commands/bbgo_hoptimize.md index b73d4bc4a..947b9f079 100644 --- a/doc/commands/bbgo_hoptimize.md +++ b/doc/commands/bbgo_hoptimize.md @@ -45,4 +45,4 @@ bbgo hoptimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_kline.md b/doc/commands/bbgo_kline.md index 684ca6e99..eefb045b2 100644 --- a/doc/commands/bbgo_kline.md +++ b/doc/commands/bbgo_kline.md @@ -42,4 +42,4 @@ bbgo kline [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_list-orders.md b/doc/commands/bbgo_list-orders.md index fca05d90b..9032b6097 100644 --- a/doc/commands/bbgo_list-orders.md +++ b/doc/commands/bbgo_list-orders.md @@ -41,4 +41,4 @@ bbgo list-orders open|closed --session SESSION --symbol SYMBOL [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_margin.md b/doc/commands/bbgo_margin.md index 6d4420453..6ea3daa8b 100644 --- a/doc/commands/bbgo_margin.md +++ b/doc/commands/bbgo_margin.md @@ -38,4 +38,4 @@ margin related history * [bbgo margin loans](bbgo_margin_loans.md) - query loans history * [bbgo margin repays](bbgo_margin_repays.md) - query repay history -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_margin_interests.md b/doc/commands/bbgo_margin_interests.md index fe99b1f64..645c604aa 100644 --- a/doc/commands/bbgo_margin_interests.md +++ b/doc/commands/bbgo_margin_interests.md @@ -41,4 +41,4 @@ bbgo margin interests --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_margin_loans.md b/doc/commands/bbgo_margin_loans.md index 707effa64..e336f2531 100644 --- a/doc/commands/bbgo_margin_loans.md +++ b/doc/commands/bbgo_margin_loans.md @@ -41,4 +41,4 @@ bbgo margin loans --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_margin_repays.md b/doc/commands/bbgo_margin_repays.md index 1572edb08..b6f9b7aec 100644 --- a/doc/commands/bbgo_margin_repays.md +++ b/doc/commands/bbgo_margin_repays.md @@ -41,4 +41,4 @@ bbgo margin repays --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_market.md b/doc/commands/bbgo_market.md index e01fe7407..aaf0435e6 100644 --- a/doc/commands/bbgo_market.md +++ b/doc/commands/bbgo_market.md @@ -40,4 +40,4 @@ bbgo market [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_optimize.md b/doc/commands/bbgo_optimize.md index 3df96f131..5dc938ed7 100644 --- a/doc/commands/bbgo_optimize.md +++ b/doc/commands/bbgo_optimize.md @@ -44,4 +44,4 @@ bbgo optimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_orderbook.md b/doc/commands/bbgo_orderbook.md index bb4f4d443..fc712c940 100644 --- a/doc/commands/bbgo_orderbook.md +++ b/doc/commands/bbgo_orderbook.md @@ -42,4 +42,4 @@ bbgo orderbook --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_orderupdate.md b/doc/commands/bbgo_orderupdate.md index dd88e0a1a..bbd40cef6 100644 --- a/doc/commands/bbgo_orderupdate.md +++ b/doc/commands/bbgo_orderupdate.md @@ -40,4 +40,4 @@ bbgo orderupdate [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_pnl.md b/doc/commands/bbgo_pnl.md index c7b28f2f3..7c0a5122b 100644 --- a/doc/commands/bbgo_pnl.md +++ b/doc/commands/bbgo_pnl.md @@ -49,4 +49,4 @@ bbgo pnl [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_run.md b/doc/commands/bbgo_run.md index 578af4b2e..667b8c8c8 100644 --- a/doc/commands/bbgo_run.md +++ b/doc/commands/bbgo_run.md @@ -51,4 +51,4 @@ bbgo run [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_submit-order.md b/doc/commands/bbgo_submit-order.md index 8a0c60292..4aeca4c37 100644 --- a/doc/commands/bbgo_submit-order.md +++ b/doc/commands/bbgo_submit-order.md @@ -46,4 +46,4 @@ bbgo submit-order --session SESSION --symbol SYMBOL --side SIDE --quantity QUANT * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_sync.md b/doc/commands/bbgo_sync.md index 26650fc61..7e0df1362 100644 --- a/doc/commands/bbgo_sync.md +++ b/doc/commands/bbgo_sync.md @@ -42,4 +42,4 @@ bbgo sync [--session=[exchange_name]] [--symbol=[pair_name]] [[--since=yyyy/mm/d * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_trades.md b/doc/commands/bbgo_trades.md index bd3edcbea..f2c0c9ac1 100644 --- a/doc/commands/bbgo_trades.md +++ b/doc/commands/bbgo_trades.md @@ -42,4 +42,4 @@ bbgo trades --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_tradeupdate.md b/doc/commands/bbgo_tradeupdate.md index 916e8d7dc..02a156a82 100644 --- a/doc/commands/bbgo_tradeupdate.md +++ b/doc/commands/bbgo_tradeupdate.md @@ -40,4 +40,4 @@ bbgo tradeupdate --session=[exchange_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_transfer-history.md b/doc/commands/bbgo_transfer-history.md index cdc5373c6..dd86d5bd8 100644 --- a/doc/commands/bbgo_transfer-history.md +++ b/doc/commands/bbgo_transfer-history.md @@ -42,4 +42,4 @@ bbgo transfer-history [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_userdatastream.md b/doc/commands/bbgo_userdatastream.md index 72df8d23c..0b83b12cb 100644 --- a/doc/commands/bbgo_userdatastream.md +++ b/doc/commands/bbgo_userdatastream.md @@ -40,4 +40,4 @@ bbgo userdatastream [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_version.md b/doc/commands/bbgo_version.md index 91e741f82..e27f0988f 100644 --- a/doc/commands/bbgo_version.md +++ b/doc/commands/bbgo_version.md @@ -39,4 +39,4 @@ bbgo version [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 From 31fb96c171fad3d1dc3192b2683ec0c2d2e64cb8 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 9 Nov 2023 12:56:18 +0800 Subject: [PATCH 29/33] bump version to v1.53.0 --- pkg/version/dev.go | 4 ++-- pkg/version/version.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/version/dev.go b/pkg/version/dev.go index df7940b6e..5b92569c2 100644 --- a/pkg/version/dev.go +++ b/pkg/version/dev.go @@ -3,6 +3,6 @@ package version -const Version = "v1.52.0-2058ce80-dev" +const Version = "v1.53.0-4c701676-dev" -const VersionGitRef = "2058ce80" +const VersionGitRef = "4c701676" diff --git a/pkg/version/version.go b/pkg/version/version.go index 1be5d7703..cdefa39ba 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -3,6 +3,6 @@ package version -const Version = "v1.52.0-2058ce80" +const Version = "v1.53.0-4c701676" -const VersionGitRef = "2058ce80" +const VersionGitRef = "4c701676" From 28c8fda2dbac80c740a84ab0775e52eb2002e2b5 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 9 Nov 2023 12:56:18 +0800 Subject: [PATCH 30/33] add v1.53.0 release note --- doc/release/v1.53.0.md | 65 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 doc/release/v1.53.0.md diff --git a/doc/release/v1.53.0.md b/doc/release/v1.53.0.md new file mode 100644 index 000000000..8654d5f4f --- /dev/null +++ b/doc/release/v1.53.0.md @@ -0,0 +1,65 @@ +[Full Changelog](https://github.com/c9s/bbgo/compare/v1.52.0...main) + + - [#1401](https://github.com/c9s/bbgo/pull/1401): STRATEGY: add liquidity maker + - [#1403](https://github.com/c9s/bbgo/pull/1403): FEATURE: [bybit] add assertion for API response + - [#1394](https://github.com/c9s/bbgo/pull/1394): FEATURE: [bitget] support query closed orders + - [#1392](https://github.com/c9s/bbgo/pull/1392): FEATURE: [bitget] add query open orders + - [#1396](https://github.com/c9s/bbgo/pull/1396): FEATURE: add ttl for position/grid2.profit stats persistence + - [#1395](https://github.com/c9s/bbgo/pull/1395): FIX: fix skip syncing active order + - [#1398](https://github.com/c9s/bbgo/pull/1398): FIX: [bybit] rm retry and add fee recover + - [#1397](https://github.com/c9s/bbgo/pull/1397): FEATURE: [bybit] to periodically fetch the fee rate + - [#1391](https://github.com/c9s/bbgo/pull/1391): FIX: [grid2] respect BaseGridNum and add a failing test case + - [#1390](https://github.com/c9s/bbgo/pull/1390): FIX: [rebalance] fix buy quantity + - [#1380](https://github.com/c9s/bbgo/pull/1380): FEATURE: [bitget] support kline subscription on stream + - [#1385](https://github.com/c9s/bbgo/pull/1385): FEATURE: [bitget] add query tickers api + - [#1376](https://github.com/c9s/bbgo/pull/1376): FEATURE: query trades from db page by page + - [#1386](https://github.com/c9s/bbgo/pull/1386): REFACTOR: [wall] refactor wall strategy with common.Strategy + - [#1382](https://github.com/c9s/bbgo/pull/1382): REFACTOR: [bitget] add rate limiter for account, ticker + - [#1384](https://github.com/c9s/bbgo/pull/1384): CHORE: minor improvements on backtest cmd + - [#1381](https://github.com/c9s/bbgo/pull/1381): DOC: grammatical errors in the README.md + - [#1377](https://github.com/c9s/bbgo/pull/1377): REFACTOR: [rebalance] submit one order at a time + - [#1378](https://github.com/c9s/bbgo/pull/1378): REFACTOR: [bitget] get symbol api + - [#1375](https://github.com/c9s/bbgo/pull/1375): DOC: grammatical error in the code_of_conduct file + - [#1374](https://github.com/c9s/bbgo/pull/1374): FIX: retry to get open orders only for 5 times and do not sync orders… + - [#1368](https://github.com/c9s/bbgo/pull/1368): FEATURE: merge grid recover and active orders recover logic + - [#1367](https://github.com/c9s/bbgo/pull/1367): DOC: fix typos in doc/development + - [#1372](https://github.com/c9s/bbgo/pull/1372): FIX: [bybit][kucoin] fix negative volume, price precision + - [#1373](https://github.com/c9s/bbgo/pull/1373): FEATURE: [xalign] adjust quantity by max amount + - [#1363](https://github.com/c9s/bbgo/pull/1363): FEATURE: [bitget] support ping/pong + - [#1370](https://github.com/c9s/bbgo/pull/1370): REFACTOR: [stream] move ping into stream level + - [#1361](https://github.com/c9s/bbgo/pull/1361): FEATURE: prepare query trades funtion for new recover + - [#1365](https://github.com/c9s/bbgo/pull/1365): FEATURE: [batch] add jumpIfEmpty opts to closed order batch query + - [#1364](https://github.com/c9s/bbgo/pull/1364): FEATURE: [batch] add a jumpIfEmpty to batch trade option + - [#1362](https://github.com/c9s/bbgo/pull/1362): DOC: Modified README.md file's language. + - [#1360](https://github.com/c9s/bbgo/pull/1360): DOC: Update CONTRIBUTING.md + - [#1351](https://github.com/c9s/bbgo/pull/1351): DOC: Update README.md + - [#1355](https://github.com/c9s/bbgo/pull/1355): REFACTOR: rename file and variable + - [#1358](https://github.com/c9s/bbgo/pull/1358): MINOR: [indicator] remove zero padding from RMA + - [#1357](https://github.com/c9s/bbgo/pull/1357): FIX: Fix duplicate RMA values and add test cases + - [#1356](https://github.com/c9s/bbgo/pull/1356): FIX: fix rma zero value issue + - [#1350](https://github.com/c9s/bbgo/pull/1350): FEATURE: [grid2] twin orderbook + - [#1353](https://github.com/c9s/bbgo/pull/1353): CHORE: go: update requestgen to v1.3.5 + - [#1349](https://github.com/c9s/bbgo/pull/1349): MINOR: remove profit entries from profit stats + - [#1352](https://github.com/c9s/bbgo/pull/1352): DOC: Fixed a typo in README.md + - [#1347](https://github.com/c9s/bbgo/pull/1347): FEATURE: [bitget] support market trade stream + - [#1344](https://github.com/c9s/bbgo/pull/1344): FEATURE: [bitget] support book stream on bitget + - [#1280](https://github.com/c9s/bbgo/pull/1280): FEATURE: [bitget] integrate QueryMarkets, QueryTicker and QueryAccount api + - [#1346](https://github.com/c9s/bbgo/pull/1346): FIX: [xnav] skip public only session + - [#1345](https://github.com/c9s/bbgo/pull/1345): FIX: [bbgo] check symbol length for injection + - [#1343](https://github.com/c9s/bbgo/pull/1343): FIX: [max] remove outdated margin fields + - [#1328](https://github.com/c9s/bbgo/pull/1328): FEATURE: recover active orders with open orders periodically + - [#1341](https://github.com/c9s/bbgo/pull/1341): REFACTOR: [random] remove adjustQuantity from config + - [#1342](https://github.com/c9s/bbgo/pull/1342): CHORE: make rightWindow possible to be set as zero + - [#1339](https://github.com/c9s/bbgo/pull/1339): FEATURE: [BYBIT] support order book depth 200 on bybit + - [#1340](https://github.com/c9s/bbgo/pull/1340): CHORE: update xfixedmaker config for backtest + - [#1335](https://github.com/c9s/bbgo/pull/1335): FEATURE: add custom private channel support to max + - [#1338](https://github.com/c9s/bbgo/pull/1338): FIX: [grid2] set max retries to 5 + - [#1337](https://github.com/c9s/bbgo/pull/1337): REFACTOR: rename randomtrader to random + - [#1327](https://github.com/c9s/bbgo/pull/1327): FIX: Fix duplicate orders caused by position risk control + - [#1331](https://github.com/c9s/bbgo/pull/1331): FEATURE: add xfixedmaker strategy + - [#1336](https://github.com/c9s/bbgo/pull/1336): FEATURE: add randomtrader strategy + - [#1332](https://github.com/c9s/bbgo/pull/1332): FEATURE: add supported interval for okex + - [#1232](https://github.com/c9s/bbgo/pull/1232): FEATURE: add forceOrder api for binance to show liquid info + - [#1334](https://github.com/c9s/bbgo/pull/1334): CHORE: [maxapi] change default http transport settings + - [#1330](https://github.com/c9s/bbgo/pull/1330): REFACTOR: Make fixedmaker simpler + - [#1312](https://github.com/c9s/bbgo/pull/1312): FEATURE: add QueryClosedOrders() and QueryTrades() for okex From cb5e305fedafbc9a9855011c9af8a09b43ce79a8 Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 6 Nov 2023 11:51:16 +0800 Subject: [PATCH 31/33] pkg/exchange: support submit order --- .../bitget/bitgetapi/v2/client_test.go | 15 +- .../bitgetapi/v2/place_order_request.go | 29 ++ .../v2/place_order_request_requestgen.go | 251 ++++++++++++++++++ pkg/exchange/bitget/convert.go | 26 ++ pkg/exchange/bitget/convert_test.go | 26 ++ pkg/exchange/bitget/exchange.go | 113 +++++++- 6 files changed, 456 insertions(+), 4 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/place_order_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/place_order_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 3d56735e9..7b178a630 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -2,11 +2,12 @@ package bitgetapi import ( "context" - "github.com/stretchr/testify/assert" "os" "strconv" "testing" + "github.com/stretchr/testify/assert" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" "github.com/c9s/bbgo/pkg/testutil" ) @@ -45,4 +46,16 @@ func TestClient(t *testing.T) { t.Logf("place order resp: %+v", req) }) + + t.Run("PlaceOrderRequest", func(t *testing.T) { + req, err := client.NewPlaceOrderRequest().Symbol("APEUSDT").OrderType(OrderTypeLimit). + Side(SideTypeSell). + Price("2"). + Size("5"). + Force(OrderForceGTC). + Do(context.Background()) + assert.NoError(t, err) + + t.Logf("place order resp: %+v", req) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/place_order_request.go b/pkg/exchange/bitget/bitgetapi/v2/place_order_request.go new file mode 100644 index 000000000..e36679dec --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/place_order_request.go @@ -0,0 +1,29 @@ +package bitgetapi + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +import ( + "github.com/c9s/requestgen" +) + +type PlaceOrderResponse struct { + OrderId string `json:"orderId"` + ClientOrderId string `json:"clientOrderId"` +} + +//go:generate PostRequest -url "/api/v2/spot/trade/place-order" -type PlaceOrderRequest -responseDataType .PlaceOrderResponse +type PlaceOrderRequest struct { + client requestgen.AuthenticatedAPIClient + symbol string `param:"symbol"` + orderType OrderType `param:"orderType"` + side SideType `param:"side"` + force OrderForce `param:"force"` + price *string `param:"price"` + size string `param:"size"` + clientOrderId *string `param:"clientOid"` +} + +func (c *Client) NewPlaceOrderRequest() *PlaceOrderRequest { + return &PlaceOrderRequest{client: c.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/place_order_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/place_order_request_requestgen.go new file mode 100644 index 000000000..a5c8c9d87 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/place_order_request_requestgen.go @@ -0,0 +1,251 @@ +// Code generated by "requestgen -method POST -responseType .APIResponse -responseDataField Data -url /api/v2/spot/trade/place-order -type PlaceOrderRequest -responseDataType .PlaceOrderResponse"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (p *PlaceOrderRequest) Symbol(symbol string) *PlaceOrderRequest { + p.symbol = symbol + return p +} + +func (p *PlaceOrderRequest) OrderType(orderType OrderType) *PlaceOrderRequest { + p.orderType = orderType + return p +} + +func (p *PlaceOrderRequest) Side(side SideType) *PlaceOrderRequest { + p.side = side + return p +} + +func (p *PlaceOrderRequest) Force(force OrderForce) *PlaceOrderRequest { + p.force = force + return p +} + +func (p *PlaceOrderRequest) Price(price string) *PlaceOrderRequest { + p.price = &price + return p +} + +func (p *PlaceOrderRequest) Size(size string) *PlaceOrderRequest { + p.size = size + return p +} + +func (p *PlaceOrderRequest) ClientOrderId(clientOrderId string) *PlaceOrderRequest { + p.clientOrderId = &clientOrderId + return p +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (p *PlaceOrderRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (p *PlaceOrderRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + symbol := p.symbol + + // assign parameter of symbol + params["symbol"] = symbol + // check orderType field -> json key orderType + orderType := p.orderType + + // TEMPLATE check-valid-values + switch orderType { + case OrderTypeLimit, OrderTypeMarket: + params["orderType"] = orderType + + default: + return nil, fmt.Errorf("orderType value %v is invalid", orderType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of orderType + params["orderType"] = orderType + // check side field -> json key side + side := p.side + + // TEMPLATE check-valid-values + switch side { + case SideTypeBuy, SideTypeSell: + params["side"] = side + + default: + return nil, fmt.Errorf("side value %v is invalid", side) + + } + // END TEMPLATE check-valid-values + + // assign parameter of side + params["side"] = side + // check force field -> json key force + force := p.force + + // TEMPLATE check-valid-values + switch force { + case OrderForceGTC, OrderForcePostOnly, OrderForceFOK, OrderForceIOC: + params["force"] = force + + default: + return nil, fmt.Errorf("force value %v is invalid", force) + + } + // END TEMPLATE check-valid-values + + // assign parameter of force + params["force"] = force + // check price field -> json key price + if p.price != nil { + price := *p.price + + // assign parameter of price + params["price"] = price + } else { + } + // check size field -> json key size + size := p.size + + // assign parameter of size + params["size"] = size + // check clientOrderId field -> json key clientOid + if p.clientOrderId != nil { + clientOrderId := *p.clientOrderId + + // assign parameter of clientOrderId + params["clientOid"] = clientOrderId + } else { + } + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (p *PlaceOrderRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := p.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if p.isVarSlice(_v) { + p.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (p *PlaceOrderRequest) GetParametersJSON() ([]byte, error) { + params, err := p.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (p *PlaceOrderRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (p *PlaceOrderRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (p *PlaceOrderRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (p *PlaceOrderRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (p *PlaceOrderRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := p.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (p *PlaceOrderRequest) Do(ctx context.Context) (*PlaceOrderResponse, error) { + + params, err := p.GetParameters() + if err != nil { + return nil, err + } + query := url.Values{} + + apiURL := "/api/v2/spot/trade/place-order" + + req, err := p.client.NewAuthenticatedRequest(ctx, "POST", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := p.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + var data PlaceOrderResponse + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return &data, nil +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 24785a81f..a4026e150 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -243,3 +243,29 @@ func processMarketBuyQuantity(filledQty, filledPrice, priceAvg, buyQty fixedpoin return fixedpoint.Zero, fmt.Errorf("failed to execute market buy quantity due to unexpected order status %s ", orderStatus) } } + +func toLocalOrderType(orderType types.OrderType) (v2.OrderType, error) { + switch orderType { + case types.OrderTypeLimit: + return v2.OrderTypeLimit, nil + + case types.OrderTypeMarket: + return v2.OrderTypeMarket, nil + + default: + return "", fmt.Errorf("order type %s not supported", orderType) + } +} + +func toLocalSide(side types.SideType) (v2.SideType, error) { + switch side { + case types.SideTypeSell: + return v2.SideTypeSell, nil + + case types.SideTypeBuy: + return v2.SideTypeBuy, nil + + default: + return "", fmt.Errorf("side type %s not supported", side) + } +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 174798074..705eb88b8 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -465,3 +465,29 @@ func Test_processMarketBuyQuantity(t *testing.T) { assert.ErrorContains(err, "xxx") }) } + +func Test_toLocalOrderType(t *testing.T) { + orderType, err := toLocalOrderType(types.OrderTypeLimit) + assert.NoError(t, err) + assert.Equal(t, v2.OrderTypeLimit, orderType) + + orderType, err = toLocalOrderType(types.OrderTypeMarket) + assert.NoError(t, err) + assert.Equal(t, v2.OrderTypeMarket, orderType) + + _, err = toLocalOrderType("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_toLocalSide(t *testing.T) { + orderType, err := toLocalSide(types.SideTypeSell) + assert.NoError(t, err) + assert.Equal(t, v2.SideTypeSell, orderType) + + orderType, err = toLocalSide(types.SideTypeBuy) + assert.NoError(t, err) + assert.Equal(t, v2.SideTypeBuy, orderType) + + _, err = toLocalOrderType("xxx") + assert.ErrorContains(t, err, "xxx") +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 298054e0a..256cc74ef 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -42,6 +42,8 @@ var ( queryOpenOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) // closedQueryOrdersRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/trade/Get-History-Orders closedQueryOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/15), 5) + // submitOrdersRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Place-Order + submitOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) ) type Exchange struct { @@ -183,9 +185,114 @@ func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, return bals, nil } +// SubmitOrder submits an order. +// +// Remark: +// 1. We support only GTC for time-in-force, because the response from queryOrder does not include time-in-force information. +// 2. For market buy orders, the size unit is quote currency, whereas the unit for order.Quantity is in base currency. +// Therefore, we need to calculate the equivalent quote currency amount based on the ticker data. +// +// Note that there is a bug in Bitget where you can place a market order with the 'post_only' option successfully, +// which should not be possible. The issue has been reported. func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (createdOrder *types.Order, err error) { - // TODO implement me - panic("implement me") + if len(order.Market.Symbol) == 0 { + return nil, fmt.Errorf("order.Market.Symbol is required: %+v", order) + } + + req := e.v2Client.NewPlaceOrderRequest() + req.Symbol(order.Market.Symbol) + + // set order type + orderType, err := toLocalOrderType(order.Type) + if err != nil { + return nil, err + } + req.OrderType(orderType) + + // set side + side, err := toLocalSide(order.Side) + if err != nil { + return nil, err + } + req.Side(side) + + // set quantity + qty := order.Quantity + // if the order is market buy, the quantity is quote coin, instead of base coin. so we need to convert it. + if order.Type == types.OrderTypeMarket && order.Side == types.SideTypeBuy { + ticker, err := e.QueryTicker(ctx, order.Market.Symbol) + if err != nil { + return nil, err + } + qty = order.Quantity.Mul(ticker.Buy) + } + req.Size(order.Market.FormatQuantity(qty)) + + // we support only GTC/PostOnly, this is because: + // 1. We support only SPOT trading. + // 2. The query oepn/closed order does not including the `force` in SPOT. + // If we support FOK/IOC, but you can't query them, that would be unreasonable. + // The other case to consider is 'PostOnly', which is a trade-off because we want to support 'xmaker'. + if order.TimeInForce != types.TimeInForceGTC { + return nil, fmt.Errorf("time-in-force %s not supported", order.TimeInForce) + } + req.Force(v2.OrderForceGTC) + // set price + if order.Type == types.OrderTypeLimit || order.Type == types.OrderTypeLimitMaker { + req.Price(order.Market.FormatPrice(order.Price)) + + if order.Type == types.OrderTypeLimitMaker { + req.Force(v2.OrderForcePostOnly) + } + } + + // set client order id + if len(order.ClientOrderID) > maxOrderIdLen { + return nil, fmt.Errorf("unexpected length of order id, got: %d", len(order.ClientOrderID)) + } + if len(order.ClientOrderID) > 0 { + req.ClientOrderId(order.ClientOrderID) + } + + if err := submitOrdersRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("place order rate limiter wait error: %w", err) + } + res, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to place order, order: %#v, err: %w", order, err) + } + + if len(res.OrderId) == 0 || (len(order.ClientOrderID) != 0 && res.ClientOrderId != order.ClientOrderID) { + return nil, fmt.Errorf("unexpected order id, resp: %#v, order: %#v", res, order) + } + + orderId := res.OrderId + ordersResp, err := e.v2Client.NewGetUnfilledOrdersRequest().OrderId(orderId).Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query open order by order id: %s, err: %w", orderId, err) + } + + switch len(ordersResp) { + case 0: + // The market order will be executed immediately, so we cannot retrieve it through the NewGetUnfilledOrdersRequest API. + // Try to get the order from the NewGetHistoryOrdersRequest API. + ordersResp, err := e.v2Client.NewGetHistoryOrdersRequest().OrderId(orderId).Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query history order by order id: %s, err: %w", orderId, err) + } + + if len(ordersResp) != 1 { + return nil, fmt.Errorf("unexpected order length, order id: %s", orderId) + } + + return toGlobalOrder(ordersResp[0]) + + case 1: + return unfilledOrderToGlobalOrder(ordersResp[0]) + + default: + return nil, fmt.Errorf("unexpected order length, order id: %s", orderId) + } } func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders []types.Order, err error) { @@ -238,7 +345,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ // ** Since and Until cannot exceed 90 days. ** // ** Since from the last 90 days can be queried. ** func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) { - if since.Sub(time.Now()) > queryMaxDuration { + if time.Since(since) > queryMaxDuration { return nil, fmt.Errorf("start time from the last 90 days can be queried, got: %s", since) } if until.Before(since) { From a26b1582308c22a15c50d67e4cc2ef3b8405bbcf Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 9 Nov 2023 10:35:53 +0800 Subject: [PATCH 32/33] pkg/exchange: support query trades --- .../bitget/bitgetapi/v2/client_test.go | 7 + .../bitget/bitgetapi/v2/get_trade_fills.go | 70 ++++++ .../v2/get_trade_fills_request_requestgen.go | 219 ++++++++++++++++++ pkg/exchange/bitget/convert.go | 60 +++++ pkg/exchange/bitget/convert_test.go | 88 +++++++ pkg/exchange/bitget/exchange.go | 69 ++++++ 6 files changed, 513 insertions(+) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 7b178a630..2e22f560d 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -58,4 +58,11 @@ func TestClient(t *testing.T) { t.Logf("place order resp: %+v", req) }) + + t.Run("GetTradeFillsRequest", func(t *testing.T) { + req, err := client.NewGetTradeFillsRequest().Symbol("APEUSDT").Do(ctx) + assert.NoError(t, err) + + t.Logf("get trade fills resp: %+v", req) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go new file mode 100644 index 000000000..358bbda27 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go @@ -0,0 +1,70 @@ +package bitgetapi + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type TradeScope string + +const ( + TradeMaker TradeScope = "maker" + TradeTaker TradeScope = "taker" +) + +type DiscountStatus string + +const ( + DiscountYes DiscountStatus = "yes" + DiscountNo DiscountStatus = "no" +) + +type TradeFee struct { + // Discount or not + Deduction DiscountStatus `json:"deduction"` + // Transaction fee coin + FeeCoin string `json:"feeCoin"` + // Total transaction fee discount + TotalDeductionFee fixedpoint.Value `json:"totalDeductionFee"` + // Total transaction fee + TotalFee fixedpoint.Value `json:"totalFee"` +} + +type Trade struct { + UserId types.StrInt64 `json:"userId"` + Symbol string `json:"symbol"` + OrderId types.StrInt64 `json:"orderId"` + TradeId types.StrInt64 `json:"tradeId"` + OrderType OrderType `json:"orderType"` + Side SideType `json:"side"` + PriceAvg fixedpoint.Value `json:"priceAvg"` + Size fixedpoint.Value `json:"size"` + Amount fixedpoint.Value `json:"amount"` + FeeDetail TradeFee `json:"feeDetail"` + TradeScope TradeScope `json:"tradeScope"` + CTime types.MillisecondTimestamp `json:"cTime"` + UTime types.MillisecondTimestamp `json:"uTime"` +} + +//go:generate GetRequest -url "/api/v2/spot/trade/fills" -type GetTradeFillsRequest -responseDataType []Trade +type GetTradeFillsRequest struct { + client requestgen.AuthenticatedAPIClient + + symbol string `param:"symbol,query"` + // Limit number default 100 max 100 + limit *string `param:"limit,query"` + // idLessThan requests the content on the page before this ID (older data), the value input should be the orderId of the corresponding interface. + idLessThan *string `param:"idLessThan,query"` + startTime *int64 `param:"startTime,query"` + endTime *int64 `param:"endTime,query"` + orderId *string `param:"orderId,query"` +} + +func (s *Client) NewGetTradeFillsRequest() *GetTradeFillsRequest { + return &GetTradeFillsRequest{client: s.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go new file mode 100644 index 000000000..2f19cf378 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go @@ -0,0 +1,219 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/trade/fills -type GetTradeFillsRequest -responseDataType []Trade"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (s *GetTradeFillsRequest) Symbol(symbol string) *GetTradeFillsRequest { + s.symbol = symbol + return s +} + +func (s *GetTradeFillsRequest) Limit(limit string) *GetTradeFillsRequest { + s.limit = &limit + return s +} + +func (s *GetTradeFillsRequest) IdLessThan(idLessThan string) *GetTradeFillsRequest { + s.idLessThan = &idLessThan + return s +} + +func (s *GetTradeFillsRequest) StartTime(startTime int64) *GetTradeFillsRequest { + s.startTime = &startTime + return s +} + +func (s *GetTradeFillsRequest) EndTime(endTime int64) *GetTradeFillsRequest { + s.endTime = &endTime + return s +} + +func (s *GetTradeFillsRequest) OrderId(orderId string) *GetTradeFillsRequest { + s.orderId = &orderId + return s +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (s *GetTradeFillsRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + symbol := s.symbol + + // assign parameter of symbol + params["symbol"] = symbol + // check limit field -> json key limit + if s.limit != nil { + limit := *s.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + // check idLessThan field -> json key idLessThan + if s.idLessThan != nil { + idLessThan := *s.idLessThan + + // assign parameter of idLessThan + params["idLessThan"] = idLessThan + } else { + } + // check startTime field -> json key startTime + if s.startTime != nil { + startTime := *s.startTime + + // assign parameter of startTime + params["startTime"] = startTime + } else { + } + // check endTime field -> json key endTime + if s.endTime != nil { + endTime := *s.endTime + + // assign parameter of endTime + params["endTime"] = endTime + } else { + } + // check orderId field -> json key orderId + if s.orderId != nil { + orderId := *s.orderId + + // assign parameter of orderId + params["orderId"] = orderId + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (s *GetTradeFillsRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (s *GetTradeFillsRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := s.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if s.isVarSlice(_v) { + s.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (s *GetTradeFillsRequest) GetParametersJSON() ([]byte, error) { + params, err := s.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (s *GetTradeFillsRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (s *GetTradeFillsRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (s *GetTradeFillsRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (s *GetTradeFillsRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (s *GetTradeFillsRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := s.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (s *GetTradeFillsRequest) Do(ctx context.Context) ([]Trade, error) { + + // no body params + var params interface{} + query, err := s.GetQueryParameters() + if err != nil { + return nil, err + } + + apiURL := "/api/v2/spot/trade/fills" + + req, err := s.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := s.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + var data []Trade + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index a4026e150..59f335730 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -109,6 +109,66 @@ func toGlobalOrderStatus(status v2.OrderStatus) (types.OrderStatus, error) { } } +func isMaker(s v2.TradeScope) (bool, error) { + switch s { + case v2.TradeMaker: + return true, nil + + case v2.TradeTaker: + return false, nil + + default: + return false, fmt.Errorf("unexpected trade scope: %s", s) + } +} + +func isFeeDiscount(s v2.DiscountStatus) (bool, error) { + switch s { + case v2.DiscountYes: + return true, nil + + case v2.DiscountNo: + return false, nil + + default: + return false, fmt.Errorf("unexpected discount status: %s", s) + } +} + +func toGlobalTrade(trade v2.Trade) (*types.Trade, error) { + side, err := toGlobalSideType(trade.Side) + if err != nil { + return nil, err + } + + isMaker, err := isMaker(trade.TradeScope) + if err != nil { + return nil, err + } + + isDiscount, err := isFeeDiscount(trade.FeeDetail.Deduction) + if err != nil { + return nil, err + } + + return &types.Trade{ + ID: uint64(trade.TradeId), + OrderID: uint64(trade.OrderId), + Exchange: types.ExchangeBitget, + Price: trade.PriceAvg, + Quantity: trade.Size, + QuoteQuantity: trade.Amount, + Symbol: trade.Symbol, + Side: side, + IsBuyer: side == types.SideTypeBuy, + IsMaker: isMaker, + Time: types.Time(trade.CTime), + Fee: trade.FeeDetail.TotalFee.Abs(), + FeeCurrency: trade.FeeDetail.FeeCoin, + FeeDiscounted: isDiscount, + }, nil +} + // unfilledOrderToGlobalOrder convert the local order to global. // // Note that the quantity unit, according official document: Base coin when orderType=limit; Quote coin when orderType=market diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 705eb88b8..470c63f31 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -491,3 +491,91 @@ func Test_toLocalSide(t *testing.T) { _, err = toLocalOrderType("xxx") assert.ErrorContains(t, err, "xxx") } + +func Test_isMaker(t *testing.T) { + isM, err := isMaker(v2.TradeTaker) + assert.NoError(t, err) + assert.False(t, isM) + + isM, err = isMaker(v2.TradeMaker) + assert.NoError(t, err) + assert.True(t, isM) + + _, err = isMaker("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_isFeeDiscount(t *testing.T) { + isDiscount, err := isFeeDiscount(v2.DiscountNo) + assert.NoError(t, err) + assert.False(t, isDiscount) + + isDiscount, err = isFeeDiscount(v2.DiscountYes) + assert.NoError(t, err) + assert.True(t, isDiscount) + + _, err = isFeeDiscount("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_toGlobalTrade(t *testing.T) { + // { + // "userId":"8672173294", + // "symbol":"APEUSDT", + // "orderId":"1104337778433757184", + // "tradeId":"1104337778504044545", + // "orderType":"limit", + // "side":"sell", + // "priceAvg":"1.4001", + // "size":"5", + // "amount":"7.0005", + // "feeDetail":{ + // "deduction":"no", + // "feeCoin":"USDT", + // "totalDeductionFee":"", + // "totalFee":"-0.0070005" + // }, + // "tradeScope":"taker", + // "cTime":"1699020564676", + // "uTime":"1699020564687" + //} + trade := v2.Trade{ + UserId: types.StrInt64(8672173294), + Symbol: "APEUSDT", + OrderId: types.StrInt64(1104337778433757184), + TradeId: types.StrInt64(1104337778504044545), + OrderType: v2.OrderTypeLimit, + Side: v2.SideTypeSell, + PriceAvg: fixedpoint.NewFromFloat(1.4001), + Size: fixedpoint.NewFromFloat(5), + Amount: fixedpoint.NewFromFloat(7.0005), + FeeDetail: v2.TradeFee{ + Deduction: "no", + FeeCoin: "USDT", + TotalDeductionFee: fixedpoint.Zero, + TotalFee: fixedpoint.NewFromFloat(-0.0070005), + }, + TradeScope: v2.TradeTaker, + CTime: types.NewMillisecondTimestampFromInt(1699020564676), + UTime: types.NewMillisecondTimestampFromInt(1699020564687), + } + + res, err := toGlobalTrade(trade) + assert.NoError(t, err) + assert.Equal(t, &types.Trade{ + ID: uint64(1104337778504044545), + OrderID: uint64(1104337778433757184), + Exchange: types.ExchangeBitget, + Price: fixedpoint.NewFromFloat(1.4001), + Quantity: fixedpoint.NewFromFloat(5), + QuoteQuantity: fixedpoint.NewFromFloat(7.0005), + Symbol: "APEUSDT", + Side: types.SideTypeSell, + IsBuyer: false, + IsMaker: false, + Time: types.Time(types.NewMillisecondTimestampFromInt(1699020564676)), + Fee: fixedpoint.NewFromFloat(0.0070005), + FeeCurrency: "USDT", + FeeDiscounted: false, + }, res) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 256cc74ef..4a5bf35ba 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -2,6 +2,7 @@ package bitget import ( "context" + "errors" "fmt" "strconv" "time" @@ -44,6 +45,8 @@ var ( closedQueryOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/15), 5) // submitOrdersRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Place-Order submitOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // queryTradeRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Get-Fills + queryTradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) ) type Exchange struct { @@ -393,3 +396,69 @@ func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) erro // TODO implement me panic("implement me") } + +// QueryTrades queries fill trades. The trade of the response is in descending order. The time-based query are typically +// using (`CTime`) as the search criteria. +// If you need to retrieve all data, please utilize the function pkg/exchange/batch.TradeBatchQuery. +// +// ** StartTime is inclusive, EndTime is exclusive. If you use the EndTime, the StartTime is required. ** +// ** StartTime and EndTime cannot exceed 90 days. ** +func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) { + if options.LastTradeID != 0 { + log.Warn("!!!BITGET EXCHANGE API NOTICE!!! The trade of response is in descending order, so the last trade id not supported.") + } + + req := e.v2Client.NewGetTradeFillsRequest() + req.Symbol(symbol) + + if options.StartTime != nil { + if time.Since(*options.StartTime) > queryMaxDuration { + return nil, fmt.Errorf("start time from the last 90 days can be queried, got: %s", options.StartTime) + } + req.StartTime(options.StartTime.UnixMilli()) + } + + if options.EndTime != nil { + if options.StartTime == nil { + return nil, errors.New("start time is required for query trades if you take end time") + } + if options.EndTime.Before(*options.StartTime) { + return nil, fmt.Errorf("end time %s before start %s", *options.EndTime, *options.StartTime) + } + if options.EndTime.Sub(*options.StartTime) > queryMaxDuration { + return nil, fmt.Errorf("start time %s and end time %s cannot greater than 90 days", options.StartTime, options.EndTime) + } + req.EndTime(options.EndTime.UnixMilli()) + } + + limit := options.Limit + if limit > queryLimit || limit <= 0 { + log.Debugf("limtit is exceeded or zero, update to %d, got: %d", queryLimit, options.Limit) + limit = queryLimit + } + req.Limit(strconv.FormatInt(limit, 10)) + + if err := queryTradeRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("trade rate limiter wait error: %w", err) + } + response, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query trades, err: %w", err) + } + + var errs error + for _, trade := range response { + res, err := toGlobalTrade(trade) + if err != nil { + errs = multierr.Append(errs, err) + continue + } + trades = append(trades, *res) + } + + if errs != nil { + return nil, errs + } + + return trades, nil +} From 639947c8b76118a3431a61bc0726062e1efd1c50 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 9 Nov 2023 10:38:32 +0800 Subject: [PATCH 33/33] pkg/exchange: support cancel order --- .../bitgetapi/v2/cancel_order_request.go | 29 +++ .../v2/cancel_order_request_requestgen.go | 196 ++++++++++++++++++ .../bitget/bitgetapi/v2/client_test.go | 13 ++ pkg/exchange/bitget/exchange.go | 52 ++++- 4 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/cancel_order_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/cancel_order_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request.go b/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request.go new file mode 100644 index 000000000..301212cd5 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request.go @@ -0,0 +1,29 @@ +package bitgetapi + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/types" +) + +type CancelOrder struct { + // OrderId are always numeric. It's confirmed with official customer service. https://t.me/bitgetOpenapi/24172 + OrderId types.StrInt64 `json:"orderId"` + ClientOrderId string `json:"clientOid"` +} + +//go:generate PostRequest -url "/api/v2/spot/trade/cancel-order" -type CancelOrderRequest -responseDataType .CancelOrder +type CancelOrderRequest struct { + client requestgen.AuthenticatedAPIClient + + symbol string `param:"symbol"` + orderId *string `param:"orderId"` + clientOrderId *string `param:"clientOid"` +} + +func (c *Client) NewCancelOrderRequest() *CancelOrderRequest { + return &CancelOrderRequest{client: c.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request_requestgen.go new file mode 100644 index 000000000..0be0e2ccb --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request_requestgen.go @@ -0,0 +1,196 @@ +// Code generated by "requestgen -method POST -responseType .APIResponse -responseDataField Data -url /api/v2/spot/trade/cancel-order -type CancelOrderRequest -responseDataType .CancelOrder"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (c *CancelOrderRequest) Symbol(symbol string) *CancelOrderRequest { + c.symbol = symbol + return c +} + +func (c *CancelOrderRequest) OrderId(orderId string) *CancelOrderRequest { + c.orderId = &orderId + return c +} + +func (c *CancelOrderRequest) ClientOrderId(clientOrderId string) *CancelOrderRequest { + c.clientOrderId = &clientOrderId + return c +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (c *CancelOrderRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (c *CancelOrderRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + symbol := c.symbol + + // assign parameter of symbol + params["symbol"] = symbol + // check orderId field -> json key orderId + if c.orderId != nil { + orderId := *c.orderId + + // assign parameter of orderId + params["orderId"] = orderId + } else { + } + // check clientOrderId field -> json key clientOid + if c.clientOrderId != nil { + clientOrderId := *c.clientOrderId + + // assign parameter of clientOrderId + params["clientOid"] = clientOrderId + } else { + } + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (c *CancelOrderRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := c.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if c.isVarSlice(_v) { + c.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (c *CancelOrderRequest) GetParametersJSON() ([]byte, error) { + params, err := c.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (c *CancelOrderRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (c *CancelOrderRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (c *CancelOrderRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (c *CancelOrderRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (c *CancelOrderRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := c.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (c *CancelOrderRequest) GetPath() string { + return "/api/v2/spot/trade/cancel-order" +} + +// Do generates the request object and send the request object to the API endpoint +func (c *CancelOrderRequest) Do(ctx context.Context) (*CancelOrder, error) { + + params, err := c.GetParameters() + if err != nil { + return nil, err + } + query := url.Values{} + + var apiURL string + + apiURL = c.GetPath() + + req, err := c.client.NewAuthenticatedRequest(ctx, "POST", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := c.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data CancelOrder + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return &data, nil +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 2e22f560d..97b836f7f 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -65,4 +65,17 @@ func TestClient(t *testing.T) { t.Logf("get trade fills resp: %+v", req) }) + + t.Run("CancelOrderRequest", func(t *testing.T) { + req, err := client.NewPlaceOrderRequest().Symbol("APEUSDT").OrderType(OrderTypeLimit). + Side(SideTypeSell). + Price("2"). + Size("5"). + Force(OrderForceGTC). + Do(context.Background()) + assert.NoError(t, err) + + resp, err := client.NewCancelOrderRequest().Symbol("APEUSDT").OrderId(req.OrderId).Do(ctx) + t.Logf("cancel order resp: %+v", resp) + }) } diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 4a5bf35ba..05a10bb26 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -47,6 +47,8 @@ var ( submitOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) // queryTradeRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Get-Fills queryTradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // cancelOrderRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/trade/Cancel-Order + cancelOrderRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) ) type Exchange struct { @@ -392,9 +394,53 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, return types.SortOrdersAscending(orders), nil } -func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) error { - // TODO implement me - panic("implement me") +func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) (errs error) { + if len(orders) == 0 { + return nil + } + + for _, order := range orders { + req := e.client.NewCancelOrderRequest() + + reqId := "" + switch { + // use the OrderID first, then the ClientOrderID + case order.OrderID > 0: + req.OrderId(strconv.FormatUint(order.OrderID, 10)) + reqId = strconv.FormatUint(order.OrderID, 10) + + case len(order.ClientOrderID) != 0: + req.ClientOrderId(order.ClientOrderID) + reqId = order.ClientOrderID + + default: + errs = multierr.Append( + errs, + fmt.Errorf("the order uuid and client order id are empty, order: %#v", order), + ) + continue + } + + req.Symbol(order.Market.Symbol) + + if err := cancelOrderRateLimiter.Wait(ctx); err != nil { + errs = multierr.Append(errs, fmt.Errorf("cancel order rate limiter wait, order id: %s, error: %w", order.ClientOrderID, err)) + continue + } + res, err := req.Do(ctx) + if err != nil { + errs = multierr.Append(errs, fmt.Errorf("failed to cancel order id: %s, err: %w", order.ClientOrderID, err)) + continue + } + + // sanity check + if res.OrderId != reqId && res.ClientOrderId != reqId { + errs = multierr.Append(errs, fmt.Errorf("order id mismatch, exp: %s, respOrderId: %s, respClientOrderId: %s", reqId, res.OrderId, res.ClientOrderId)) + continue + } + } + + return errs } // QueryTrades queries fill trades. The trade of the response is in descending order. The time-based query are typically