FEATURE: recover the grid when there is error at grid opening

pkg/strategy/grid2/grid_order_states.go

@@ -0,0 +1,81 @@
+package grid2
+
+import (
+	"context"
+	"strings"
+
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+	"go.uber.org/multierr"
+)
+
+type GridOrder struct {
+	OrderID       uint64           `json:"orderID"`
+	ClientOrderID string           `json:"clientOrderID"`
+	Side          types.SideType   `json:"side"`
+	Price         fixedpoint.Value `json:"price"`
+	Quantity      fixedpoint.Value `json:"quantity"`
+}
+
+type GridOrderStates struct {
+	Orders map[fixedpoint.Value]GridOrder `json:"orders"`
+}
+
+func newGridOrderStates() *GridOrderStates {
+	return &GridOrderStates{
+		Orders: make(map[fixedpoint.Value]GridOrder),
+	}
+}
+
+func (s *GridOrderStates) AddCreatedOrders(createdOrders ...types.Order) {
+	for _, createdOrder := range createdOrders {
+		s.Orders[createdOrder.Price] = GridOrder{
+			OrderID:       createdOrder.OrderID,
+			ClientOrderID: createdOrder.ClientOrderID,
+			Side:          createdOrder.Side,
+			Price:         createdOrder.Price,
+			Quantity:      createdOrder.Quantity,
+		}
+	}
+}
+
+func (s *GridOrderStates) AddSubmitOrders(submitOrders ...types.SubmitOrder) {
+	for _, submitOrder := range submitOrders {
+		s.Orders[submitOrder.Price] = GridOrder{
+			ClientOrderID: submitOrder.ClientOrderID,
+			Side:          submitOrder.Side,
+			Price:         submitOrder.Price,
+			Quantity:      submitOrder.Quantity,
+		}
+	}
+}
+
+func (s *GridOrderStates) GetFailedOrdersWhenGridOpening(ctx context.Context, orderQueryService types.ExchangeOrderQueryService) ([]GridOrder, error) {
+	var failed []GridOrder
+	var errs error
+
+	for _, order := range s.Orders {
+		if order.OrderID == 0 {
+			_, err := orderQueryService.QueryOrder(ctx, types.OrderQuery{
+				ClientOrderID: order.ClientOrderID,
+			})
+
+			if err != nil {
+				// error handle
+				if strings.Contains(err.Error(), "resource not found") {
+					// not found error, need to re-place this order
+					// if order not found: {"success":false,"error":{"code":404,"message":"resource not found"}}
+					failed = append(failed, order)
+				} else {
+					// other error
+					// need to log the error and stop
+					errs = multierr.Append(errs, err)
+				}
+
+				continue
+			}
+		}
+	}
+
+	return failed, errs
+}

pkg/strategy/grid2/strategy.go

@@ -164,6 +164,7 @@ type Strategy struct {
 
 	SkipSpreadCheck                    bool `json:"skipSpreadCheck"`
 	RecoverGridByScanningTrades        bool `json:"recoverGridByScanningTrades"`
+	RecoverFailedOrdersWhenGridOpening bool `json:"recoverFailedOrdersWhenGridOpening"`
 
 	// Debug enables the debug mode
 	Debug bool `json:"debug"`
@@ -171,6 +172,9 @@ type Strategy struct {
 	GridProfitStats *GridProfitStats `persistence:"grid_profit_stats"`
 	Position        *types.Position  `persistence:"position"`
 
+	// this is used to check all generated orders are placed
+	GridOrderStates *GridOrderStates `persistence:"grid_order_states"`
+
 	// ExchangeSession is an injection field
 	ExchangeSession *bbgo.ExchangeSession
 
@@ -1068,6 +1072,10 @@ func (s *Strategy) openGrid(ctx context.Context, session *bbgo.ExchangeSession)
 		return err
 	}
 
+	// use grid order states to check the orders are placed when opening grid
+	s.GridOrderStates = newGridOrderStates()
+	s.GridOrderStates.AddSubmitOrders(submitOrders...)
+
 	s.debugGridOrders(submitOrders, lastPrice)
 
 	writeCtx := s.getWriteContext(ctx)
@@ -1078,6 +1086,8 @@ func (s *Strategy) openGrid(ctx context.Context, session *bbgo.ExchangeSession)
 		return err2
 	}
 
+	s.GridOrderStates.AddCreatedOrders(createdOrders...)
+
 	// try to always emit grid ready
 	defer s.EmitGridReady()
 
@@ -1230,6 +1240,26 @@ func (s *Strategy) debugOrders(desc string, orders []types.Order) {
 	s.logger.Infof(sb.String())
 }
 
+func (s *Strategy) debugSubmitOrders(desc string, orders []types.SubmitOrder) {
+	if !s.Debug {
+		return
+	}
+
+	var sb strings.Builder
+
+	if desc == "" {
+		desc = "ORDERS"
+	}
+
+	sb.WriteString(desc + " [\n")
+	for i, order := range orders {
+		sb.WriteString(fmt.Sprintf("  - %d) %s\n", i, order.String()))
+	}
+	sb.WriteString("]")
+
+	s.logger.Infof(sb.String())
+}
+
 func (s *Strategy) debugGridProfitStats(trigger string) {
 	if !s.Debug {
 		return
@@ -1237,7 +1267,6 @@ func (s *Strategy) debugGridProfitStats(trigger string) {
 
 	stats := *s.GridProfitStats
 	// ProfitEntries may have too many profits, make it nil to readable
-	stats.ProfitEntries = nil
 	b, err := json.Marshal(stats)
 	if err != nil {
 		s.logger.WithError(err).Errorf("[%s] failed to debug grid profit stats", trigger)
@@ -1936,6 +1965,15 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.
 
 func (s *Strategy) startProcess(ctx context.Context, session *bbgo.ExchangeSession) {
 	s.debugGridProfitStats("startProcess")
+	if s.RecoverFailedOrdersWhenGridOpening {
+		s.logger.Info("recover failed orders when grid opening")
+		if err := s.recoverFailedOrdersWhenGridOpening(ctx); err != nil {
+			s.logger.WithError(err).Error("failed to start process, recover failed orders when grid opening error")
+			s.EmitGridError(errors.Wrapf(err, "failed to start process, recover failed orders when grid opening error"))
+			return
+		}
+	}
+
 	if s.RecoverOrdersWhenStart {
 		// do recover only when triggerPrice is not set and not in the back-test mode
 		s.logger.Infof("recoverWhenStart is set, trying to recover grid orders...")
@@ -1954,6 +1992,44 @@ func (s *Strategy) startProcess(ctx context.Context, session *bbgo.ExchangeSessi
 	}
 }
 
+func (s *Strategy) recoverFailedOrdersWhenGridOpening(ctx context.Context) error {
+	if s.GridOrderStates == nil {
+		return nil
+	}
+
+	failedOrders, err := s.GridOrderStates.GetFailedOrdersWhenGridOpening(ctx, s.orderQueryService)
+	if err != nil {
+		return errors.Wrapf(err, "failed to get failed orders")
+	}
+
+	var submitOrders []types.SubmitOrder
+	for _, failedOrder := range failedOrders {
+		submitOrders = append(submitOrders, types.SubmitOrder{
+			Symbol:        s.Symbol,
+			Type:          types.OrderTypeLimit,
+			Side:          failedOrder.Side,
+			Price:         failedOrder.Price,
+			Quantity:      failedOrder.Quantity,
+			Market:        s.Market,
+			TimeInForce:   types.TimeInForceGTC,
+			Tag:           orderTag,
+			GroupID:       s.OrderGroupID,
+			ClientOrderID: failedOrder.ClientOrderID,
+		})
+	}
+
+	s.debugSubmitOrders("RECOVER FAILED ORDERS WHEN GRID OPENING", submitOrders)
+
+	writeCtx := s.getWriteContext(ctx)
+	createdOrders, err := s.orderExecutor.SubmitOrders(writeCtx, submitOrders...)
+	if err != nil {
+		return errors.Wrapf(err, "failed to submit orders")
+	}
+
+	s.GridOrderStates.AddCreatedOrders(createdOrders...)
+	return nil
+}
+
 func (s *Strategy) recoverGrid(ctx context.Context, session *bbgo.ExchangeSession) error {
 	if s.RecoverGridByScanningTrades {
 		s.debugLog("recovering grid by scanning trades")