Merge pull request #682 from c9s/fix/backtest-order-test

fix: fix duplicated filled order update callbacks in backtest

pkg/backtest/matching.go

@@ -146,6 +146,7 @@ func (m *SimplePriceMatching) PlaceOrder(o types.SubmitOrder) (closedOrders *typ
 	order := m.newOrder(o, orderID)
 
 	if o.Type == types.OrderTypeMarket {
+		// emit the order update for Status:New
 		m.EmitOrderUpdate(order)
 
 		// copy the order object to avoid side effect (for different callbacks)
@@ -160,12 +161,13 @@ func (m *SimplePriceMatching) PlaceOrder(o types.SubmitOrder) (closedOrders *typ
 		order2.ExecutedQuantity = order2.Quantity
 		order2.Price = price
 		order2.IsWorking = false
-		m.EmitOrderUpdate(order2)
 
+		// let the exchange emit the "FILLED" order update (we need the closed order)
+		// m.EmitOrderUpdate(order2)
 		return &order2, &trade, nil
 	}
 
-	// for limit maker orders
+	// For limit maker orders (open status)
 	// TODO: handle limit taker order
 	switch o.Side {
 
@@ -180,8 +182,7 @@ func (m *SimplePriceMatching) PlaceOrder(o types.SubmitOrder) (closedOrders *typ
 		m.mu.Unlock()
 	}
 
-	m.EmitOrderUpdate(order)
-
+	m.EmitOrderUpdate(order) // emit order New status
 	return &order, nil, nil
 }
 

pkg/backtest/matching_test.go

@@ -21,6 +21,72 @@ func newLimitOrder(symbol string, side types.SideType, price, quantity float64)
 	}
 }
 
+func TestSimplePriceMatching_orderUpdate(t *testing.T) {
+	account := &types.Account{
+		MakerFeeRate: fixedpoint.NewFromFloat(0.075 * 0.01),
+		TakerFeeRate: fixedpoint.NewFromFloat(0.075 * 0.01),
+	}
+	account.UpdateBalances(types.BalanceMap{
+		"USDT": {Currency: "USDT", Available: fixedpoint.NewFromFloat(10000.0)},
+	})
+	market := types.Market{
+		Symbol:          "BTCUSDT",
+		PricePrecision:  8,
+		VolumePrecision: 8,
+		QuoteCurrency:   "USDT",
+		BaseCurrency:    "BTC",
+		MinNotional:     fixedpoint.MustNewFromString("0.001"),
+		MinAmount:       fixedpoint.MustNewFromString("10.0"),
+		MinQuantity:     fixedpoint.MustNewFromString("0.001"),
+	}
+
+	t1 := time.Date(2021, 7, 1, 0, 0, 0, 0, time.UTC)
+	engine := &SimplePriceMatching{
+		Account:     account,
+		Market:      market,
+		CurrentTime: t1,
+	}
+
+	orderUpdateCnt := 0
+	orderUpdateNewStatusCnt := 0
+	orderUpdateFilledStatusCnt := 0
+	var lastOrder types.Order
+	engine.OnOrderUpdate(func(order types.Order) {
+		lastOrder = order
+
+		orderUpdateCnt++
+		switch order.Status {
+		case types.OrderStatusNew:
+			orderUpdateNewStatusCnt++
+
+		case types.OrderStatusFilled:
+			orderUpdateFilledStatusCnt++
+
+		}
+	})
+
+	_, _, err := engine.PlaceOrder(newLimitOrder("BTCUSDT", types.SideTypeBuy, 24000.0, 0.1))
+	assert.NoError(t, err)
+	assert.Equal(t, 1, orderUpdateCnt)             // should got new status
+	assert.Equal(t, 1, orderUpdateNewStatusCnt)    // should got new status
+	assert.Equal(t, 0, orderUpdateFilledStatusCnt) // should got new status
+	assert.Equal(t, types.OrderStatusNew, lastOrder.Status)
+	assert.Equal(t, fixedpoint.NewFromFloat(0.0), lastOrder.ExecutedQuantity)
+
+	t2 := t1.Add(time.Minute)
+
+	// should match 25000, 24000
+	k := newKLine("BTCUSDT", types.Interval1m, t2, 26000, 27000, 23000, 25000)
+	engine.processKLine(k)
+
+	assert.Equal(t, 2, orderUpdateCnt)             // should got new and filled
+	assert.Equal(t, 1, orderUpdateNewStatusCnt)    // should got new status
+	assert.Equal(t, 1, orderUpdateFilledStatusCnt) // should got new status
+	assert.Equal(t, types.OrderStatusFilled, lastOrder.Status)
+	assert.Equal(t, "0.1", lastOrder.ExecutedQuantity.String())
+	assert.Equal(t, lastOrder.Quantity.String(), lastOrder.ExecutedQuantity.String())
+}
+
 func TestSimplePriceMatching_processKLine(t *testing.T) {
 	account := &types.Account{
 		MakerFeeRate: fixedpoint.NewFromFloat(0.075 * 0.01),

pkg/bbgo/order_execution.go

@@ -102,9 +102,9 @@ func (e *ExchangeOrderExecutor) SubmitOrders(ctx context.Context, orders ...type
 		// pass submit order as an interface object.
 		channel, ok := e.RouteObject(&order)
 		if ok {
-			e.NotifyTo(channel, ":memo: Submitting %s %s %s order with quantity: %f", order.Symbol, order.Type, order.Side, order.Quantity, order)
+			e.NotifyTo(channel, ":memo: Submitting %s %s %s order with quantity: %f", order.Symbol, order.Type, order.Side, order.Quantity.Float64(), order)
 		} else {
-			e.Notify(":memo: Submitting %s %s %s order with quantity: %f", order.Symbol, order.Type, order.Side, order.Quantity, order)
+			e.Notify(":memo: Submitting %s %s %s order with quantity: %f", order.Symbol, order.Type, order.Side, order.Quantity.Float64(), order)
 		}
 
 		log.Infof("submitting order: %s", order.String())

pkg/cmd/builtin.go

@@ -5,6 +5,7 @@ import (
 	_ "github.com/c9s/bbgo/pkg/strategy/autoborrow"
 	_ "github.com/c9s/bbgo/pkg/strategy/bollgrid"
 	_ "github.com/c9s/bbgo/pkg/strategy/bollmaker"
+	_ "github.com/c9s/bbgo/pkg/strategy/dca"
 	_ "github.com/c9s/bbgo/pkg/strategy/emastop"
 	_ "github.com/c9s/bbgo/pkg/strategy/etf"
 	_ "github.com/c9s/bbgo/pkg/strategy/ewoDgtrd"

pkg/strategy/dca/strategy.go

@@ -0,0 +1,248 @@
+package dca
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/c9s/bbgo/pkg/bbgo"
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+const ID = "dca"
+
+var log = logrus.WithField("strategy", ID)
+
+func init() {
+	bbgo.RegisterStrategy(ID, &Strategy{})
+}
+
+type BudgetPeriod string
+
+const (
+	BudgetPeriodDay   BudgetPeriod = "day"
+	BudgetPeriodWeek  BudgetPeriod = "week"
+	BudgetPeriodMonth BudgetPeriod = "month"
+)
+
+func (b BudgetPeriod) Duration() time.Duration {
+	var period time.Duration
+	switch b {
+	case BudgetPeriodDay:
+		period = 24 * time.Hour
+
+	case BudgetPeriodWeek:
+		period = 24 * time.Hour * 7
+
+	case BudgetPeriodMonth:
+		period = 24 * time.Hour * 30
+
+	}
+
+	return period
+}
+
+// Strategy is the Dollar-Cost-Average strategy
+type Strategy struct {
+	*bbgo.Graceful
+	*bbgo.Notifiability
+	*bbgo.Persistence
+
+	Environment *bbgo.Environment
+	Symbol      string `json:"symbol"`
+	Market      types.Market
+
+	// BudgetPeriod is how long your budget quota will be reset.
+	// day, week, month
+	BudgetPeriod BudgetPeriod `json:"budgetPeriod"`
+
+	// Budget is the amount you invest per budget period
+	Budget fixedpoint.Value `json:"budget"`
+
+	// InvestmentInterval is the interval of each investment
+	InvestmentInterval types.Interval `json:"investmentInterval"`
+
+	budgetPerInvestment fixedpoint.Value
+
+	Position              *types.Position    `persistence:"position"`
+	ProfitStats           *types.ProfitStats `persistence:"profit_stats"`
+	BudgetQuota           fixedpoint.Value   `persistence:"budget_quota"`
+	BudgetPeriodStartTime time.Time          `persistence:"budget_period_start_time"`
+
+	activeMakerOrders *bbgo.ActiveOrderBook
+	orderStore        *bbgo.OrderStore
+	tradeCollector    *bbgo.TradeCollector
+
+	session *bbgo.ExchangeSession
+
+	bbgo.StrategyController
+}
+
+func (s *Strategy) ID() string {
+	return ID
+}
+
+func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) {
+	session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.InvestmentInterval})
+}
+
+func (s *Strategy) submitOrders(ctx context.Context, orderExecutor bbgo.OrderExecutor, submitOrders ...types.SubmitOrder) {
+	createdOrders, err := orderExecutor.SubmitOrders(ctx, submitOrders...)
+	if err != nil {
+		log.WithError(err).Errorf("can not place orders")
+	}
+
+	s.orderStore.Add(createdOrders...)
+	s.activeMakerOrders.Add(createdOrders...)
+	s.tradeCollector.Process()
+}
+
+func (s *Strategy) ClosePosition(ctx context.Context, percentage fixedpoint.Value) error {
+	base := s.Position.GetBase()
+	if base.IsZero() {
+		return fmt.Errorf("no opened %s position", s.Position.Symbol)
+	}
+
+	// make it negative
+	quantity := base.Mul(percentage).Abs()
+	side := types.SideTypeBuy
+	if base.Sign() > 0 {
+		side = types.SideTypeSell
+	}
+
+	if quantity.Compare(s.Market.MinQuantity) < 0 {
+		return fmt.Errorf("order quantity %v is too small, less than %v", quantity, s.Market.MinQuantity)
+	}
+
+	submitOrder := types.SubmitOrder{
+		Symbol:   s.Symbol,
+		Side:     side,
+		Type:     types.OrderTypeMarket,
+		Quantity: quantity,
+		Market:   s.Market,
+	}
+
+	// s.Notify("Submitting %s %s order to close position by %v", s.Symbol, side.String(), percentage, submitOrder)
+
+	createdOrders, err := s.session.Exchange.SubmitOrders(ctx, submitOrder)
+	if err != nil {
+		log.WithError(err).Errorf("can not place position close order")
+	}
+
+	s.orderStore.Add(createdOrders...)
+	s.activeMakerOrders.Add(createdOrders...)
+	s.tradeCollector.Process()
+	return err
+}
+
+func (s *Strategy) InstanceID() string {
+	return fmt.Sprintf("%s:%s", ID, s.Symbol)
+}
+
+// check if position can be close or not
+func canClosePosition(position *types.Position, signal fixedpoint.Value, price fixedpoint.Value) bool {
+	return !signal.IsZero() && position.IsShort() && !position.IsDust(price)
+}
+
+func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error {
+	// initial required information
+	s.session = session
+
+	s.activeMakerOrders = bbgo.NewActiveOrderBook(s.Symbol)
+	s.activeMakerOrders.BindStream(session.UserDataStream)
+
+	s.orderStore = bbgo.NewOrderStore(s.Symbol)
+	s.orderStore.BindStream(session.UserDataStream)
+
+	if s.Position == nil {
+		s.Position = types.NewPositionFromMarket(s.Market)
+	}
+
+	if s.ProfitStats == nil {
+		s.ProfitStats = types.NewProfitStats(s.Market)
+	}
+
+	instanceID := s.InstanceID()
+
+	if s.BudgetQuota.IsZero() {
+		s.BudgetQuota = s.Budget
+	}
+
+	numOfInvestmentPerPeriod := fixedpoint.NewFromFloat(float64(s.BudgetPeriod.Duration()) / float64(s.InvestmentInterval.Duration()))
+	s.budgetPerInvestment = s.Budget.Div(numOfInvestmentPerPeriod)
+
+	// Always update the position fields
+	s.Position.Strategy = ID
+	s.Position.StrategyInstanceID = instanceID
+
+	s.tradeCollector = bbgo.NewTradeCollector(s.Symbol, s.Position, s.orderStore)
+	s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) {
+		s.Notifiability.Notify(trade)
+		s.ProfitStats.AddTrade(trade)
+
+		if profit.Compare(fixedpoint.Zero) == 0 {
+			s.Environment.RecordPosition(s.Position, trade, nil)
+		} else {
+			log.Infof("%s generated profit: %v", s.Symbol, profit)
+			p := s.Position.NewProfit(trade, profit, netProfit)
+			p.Strategy = ID
+			p.StrategyInstanceID = instanceID
+			s.Notify(&p)
+
+			s.ProfitStats.AddProfit(p)
+			s.Notify(&s.ProfitStats)
+
+			s.Environment.RecordPosition(s.Position, trade, &p)
+		}
+	})
+
+	s.tradeCollector.OnTrade(func(trade types.Trade, profit fixedpoint.Value, netProfit fixedpoint.Value) {
+		s.BudgetQuota = s.BudgetQuota.Sub(trade.QuoteQuantity)
+	})
+
+	s.tradeCollector.OnPositionUpdate(func(position *types.Position) {
+		log.Infof("position changed: %s", s.Position)
+		s.Notify(s.Position)
+	})
+
+	s.tradeCollector.BindStream(session.UserDataStream)
+
+	session.UserDataStream.OnStart(func() {})
+	session.MarketDataStream.OnKLine(func(kline types.KLine) {})
+	session.MarketDataStream.OnKLineClosed(func(kline types.KLine) {
+		if kline.Symbol != s.Symbol || kline.Interval != s.InvestmentInterval {
+			return
+		}
+
+		if s.BudgetPeriodStartTime == (time.Time{}) {
+			s.BudgetPeriodStartTime = kline.StartTime.Time().Truncate(time.Minute)
+		}
+
+		if kline.EndTime.Time().Sub(s.BudgetPeriodStartTime) >= s.BudgetPeriod.Duration() {
+			// reset budget quota
+			s.BudgetQuota = s.Budget
+			s.BudgetPeriodStartTime = kline.StartTime.Time()
+		}
+
+		// check if we have quota
+		if s.BudgetQuota.Compare(s.budgetPerInvestment) <= 0 {
+			return
+		}
+
+		price := kline.Close
+		quantity := s.budgetPerInvestment.Div(price)
+
+		s.submitOrders(ctx, orderExecutor, types.SubmitOrder{
+			Symbol:   s.Symbol,
+			Side:     types.SideTypeBuy,
+			Type:     types.OrderTypeMarket,
+			Quantity: quantity,
+			Market:   s.Market,
+		})
+	})
+
+	return nil
+}