From 8e20a5506068aab8c445ff75f660d43add08dc01 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Dec 2022 14:48:30 +0800 Subject: [PATCH 01/13] grid2: refactor recover functions to replayOrderHistory and reuse scanMissingPinPrices --- pkg/strategy/grid2/strategy.go | 111 +++++++++++++++++---------------- 1 file changed, 57 insertions(+), 54 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 2c397ee45..babfcd683 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -993,9 +993,7 @@ func (s *Strategy) checkMinimalQuoteInvestment() error { return nil } -func (s *Strategy) recoverGrid(ctx context.Context, historyService types.ExchangeTradeHistoryService, openOrders []types.Order) error { - grid := s.newGrid() - +func buildGridPriceMap(grid *Grid) PriceMap { // Add all open orders to the local order book gridPriceMap := make(PriceMap) for _, pin := range grid.Pins { @@ -1003,6 +1001,15 @@ func (s *Strategy) recoverGrid(ctx context.Context, historyService types.Exchang gridPriceMap[price.String()] = price } + return gridPriceMap +} + +func (s *Strategy) recoverGrid(ctx context.Context, historyService types.ExchangeTradeHistoryService, openOrders []types.Order) error { + grid := s.newGrid() + + // Add all open orders to the local order book + gridPriceMap := buildGridPriceMap(grid) + lastOrderID := uint64(1) now := time.Now() firstOrderTime := now.AddDate(0, 0, -7) @@ -1037,8 +1044,47 @@ func (s *Strategy) recoverGrid(ctx context.Context, historyService types.Exchang // Note that for MAX Exchange, the order history API only uses fromID parameter to query history order. // The time range does not matter. - startTime := firstOrderTime - endTime := now + // TODO: handle context correctly + if err := s.replayOrderHistory(ctx, grid, orderBook, historyService, firstOrderTime, now, lastOrderID); err != nil { + return err + } + + debugOrderBook(orderBook, grid.Pins) + + tmpOrders := orderBook.Orders() + + // if all orders on the order book are active orders, we don't need to recover. + if isCompleteGridOrderBook(orderBook, s.GridNum) { + s.logger.Infof("GRID RECOVER: all orders are active orders, do not need recover") + return nil + } + + // for reverse order recovering, we need the orders to be sort by update time ascending-ly + types.SortOrdersUpdateTimeAscending(tmpOrders) + + if len(tmpOrders) > 1 && len(tmpOrders) == int(s.GridNum)+1 { + // remove the latest updated order because it's near the empty slot + tmpOrders = tmpOrders[:len(tmpOrders)-1] + } + + // we will only submit reverse orders for filled orders + filledOrders := ordersFilled(tmpOrders) + + s.logger.Infof("GRID RECOVER: found %d filled grid orders", len(filledOrders)) + + s.grid = grid + for _, o := range filledOrders { + s.processFilledOrder(o) + } + + s.logger.Infof("GRID RECOVER COMPLETE") + + debugOrderBook(s.orderExecutor.ActiveMakerOrders(), grid.Pins) + return nil +} + +func (s *Strategy) replayOrderHistory(ctx context.Context, grid *Grid, orderBook *bbgo.ActiveOrderBook, historyService types.ExchangeTradeHistoryService, startTime, endTime time.Time, lastOrderID uint64) error { + gridPriceMap := buildGridPriceMap(grid) // a simple guard, in reality, this startTime is not possible to exceed the endTime // because the queries closed orders might still in the range. @@ -1090,45 +1136,13 @@ func (s *Strategy) recoverGrid(ctx context.Context, historyService types.Exchang } } - missingPrices := scanMissingGridOrders(orderBook, grid) + missingPrices := scanMissingPinPrices(orderBook, grid.Pins) if len(missingPrices) == 0 { s.logger.Infof("GRID RECOVER: no missing grid prices, stop re-playing order history") break } } - debugOrderBook(orderBook, grid.Pins) - - tmpOrders := orderBook.Orders() - - // if all orders on the order book are active orders, we don't need to recover. - if isCompleteGridOrderBook(orderBook, s.GridNum) { - s.logger.Infof("GRID RECOVER: all orders are active orders, do not need recover") - return nil - } - - // for reverse order recovering, we need the orders to be sort by update time ascending-ly - types.SortOrdersUpdateTimeAscending(tmpOrders) - - if len(tmpOrders) > 1 && len(tmpOrders) == int(s.GridNum)+1 { - // remove the latest updated order because it's near the empty slot - tmpOrders = tmpOrders[:len(tmpOrders)-1] - } - - // we will only submit reverse orders for filled orders - filledOrders := ordersFilled(tmpOrders) - - s.logger.Infof("GRID RECOVER: found %d filled grid orders", len(filledOrders)) - - s.grid = grid - for _, o := range filledOrders { - s.processFilledOrder(o) - } - - s.logger.Infof("GRID RECOVER COMPLETE") - - debugOrderBook(s.orderExecutor.ActiveMakerOrders(), grid.Pins) - return nil } @@ -1178,18 +1192,8 @@ func ordersAny(orders []types.Order, f func(o types.Order) bool) bool { func debugOrderBook(b *bbgo.ActiveOrderBook, pins []Pin) { fmt.Println("================== GRID ORDERS ==================") - // scan missing orders - missing := 0 - for i := len(pins) - 1; i >= 0; i-- { - pin := pins[i] - price := fixedpoint.Value(pin) - existingOrder := b.Lookup(func(o types.Order) bool { - return o.Price.Eq(price) - }) - if existingOrder == nil { - missing++ - } - } + missingPins := scanMissingPinPrices(b, pins) + missing := len(missingPins) for i := len(pins) - 1; i >= 0; i-- { pin := pins[i] @@ -1258,15 +1262,14 @@ func scanOrderCreationTimeRange(orders []types.Order) (time.Time, time.Time, boo return firstOrderTime, lastOrderTime, true } -// scanMissingGridOrders finds the missing grid order prices -func scanMissingGridOrders(orderBook *bbgo.ActiveOrderBook, grid *Grid) PriceMap { +// scanMissingPinPrices finds the missing grid order prices +func scanMissingPinPrices(orderBook *bbgo.ActiveOrderBook, pins []Pin) PriceMap { // Add all open orders to the local order book gridPrices := make(PriceMap) missingPrices := make(PriceMap) - for _, pin := range grid.Pins { + for _, pin := range pins { price := fixedpoint.Value(pin) gridPrices[price.String()] = price - existingOrder := orderBook.Lookup(func(o types.Order) bool { return o.Price.Compare(price) == 0 }) From a46b3fe90840bc99d45132d77061cc1bdc4f726b Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Dec 2022 14:52:01 +0800 Subject: [PATCH 02/13] grid2: improve debugGrid func --- pkg/strategy/grid2/strategy.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index babfcd683..948249d3d 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1049,7 +1049,7 @@ func (s *Strategy) recoverGrid(ctx context.Context, historyService types.Exchang return err } - debugOrderBook(orderBook, grid.Pins) + debugGrid(grid, orderBook) tmpOrders := orderBook.Orders() @@ -1079,7 +1079,7 @@ func (s *Strategy) recoverGrid(ctx context.Context, historyService types.Exchang s.logger.Infof("GRID RECOVER COMPLETE") - debugOrderBook(s.orderExecutor.ActiveMakerOrders(), grid.Pins) + debugGrid(grid, s.orderExecutor.ActiveMakerOrders()) return nil } @@ -1189,10 +1189,11 @@ func ordersAny(orders []types.Order, f func(o types.Order) bool) bool { return false } -func debugOrderBook(b *bbgo.ActiveOrderBook, pins []Pin) { +func debugGrid(grid *Grid, book *bbgo.ActiveOrderBook) { fmt.Println("================== GRID ORDERS ==================") - missingPins := scanMissingPinPrices(b, pins) + pins := grid.Pins + missingPins := scanMissingPinPrices(book, pins) missing := len(missingPins) for i := len(pins) - 1; i >= 0; i-- { @@ -1201,7 +1202,7 @@ func debugOrderBook(b *bbgo.ActiveOrderBook, pins []Pin) { fmt.Printf("%s -> ", price.String()) - existingOrder := b.Lookup(func(o types.Order) bool { + existingOrder := book.Lookup(func(o types.Order) bool { return o.Price.Eq(price) }) From 6b7515098365f813d945f4b4c0d2dfedf8e14f83 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Dec 2022 15:58:02 +0800 Subject: [PATCH 03/13] refactor order related functions into core api --- pkg/strategy/grid2/strategy.go | 37 ++-------------------------------- pkg/types/order.go | 33 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 948249d3d..f2e59f639 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1068,7 +1068,7 @@ func (s *Strategy) recoverGrid(ctx context.Context, historyService types.Exchang } // we will only submit reverse orders for filled orders - filledOrders := ordersFilled(tmpOrders) + filledOrders := types.OrdersFilled(tmpOrders) s.logger.Infof("GRID RECOVER: found %d filled grid orders", len(filledOrders)) @@ -1146,49 +1146,16 @@ func (s *Strategy) replayOrderHistory(ctx context.Context, grid *Grid, orderBook return nil } -func isActiveOrder(o types.Order) bool { - return o.Status == types.OrderStatusNew || o.Status == types.OrderStatusPartiallyFilled -} - func isCompleteGridOrderBook(orderBook *bbgo.ActiveOrderBook, gridNum int64) bool { tmpOrders := orderBook.Orders() - if len(tmpOrders) == int(gridNum) && ordersAll(tmpOrders, isActiveOrder) { + if len(tmpOrders) == int(gridNum) && types.OrdersAll(tmpOrders, types.IsActiveOrder) { return true } return false } -func ordersFilled(in []types.Order) (out []types.Order) { - for _, o := range in { - switch o.Status { - case types.OrderStatusFilled: - o2 := o - out = append(out, o2) - } - } - return out -} - -func ordersAll(orders []types.Order, f func(o types.Order) bool) bool { - for _, o := range orders { - if !f(o) { - return false - } - } - return true -} - -func ordersAny(orders []types.Order, f func(o types.Order) bool) bool { - for _, o := range orders { - if f(o) { - return true - } - } - return false -} - func debugGrid(grid *Grid, book *bbgo.ActiveOrderBook) { fmt.Println("================== GRID ORDERS ==================") diff --git a/pkg/types/order.go b/pkg/types/order.go index 4444bac87..82456caf9 100644 --- a/pkg/types/order.go +++ b/pkg/types/order.go @@ -394,3 +394,36 @@ func (o Order) SlackAttachment() slack.Attachment { Footer: strings.ToLower(o.Exchange.String()) + templateutil.Render(" creation time {{ . }}", o.CreationTime.Time().Format(time.StampMilli)), } } + +func OrdersFilled(in []Order) (out []Order) { + for _, o := range in { + switch o.Status { + case OrderStatusFilled: + o2 := o + out = append(out, o2) + } + } + return out +} + +func OrdersAll(orders []Order, f func(o Order) bool) bool { + for _, o := range orders { + if !f(o) { + return false + } + } + return true +} + +func OrdersAny(orders []Order, f func(o Order) bool) bool { + for _, o := range orders { + if f(o) { + return true + } + } + return false +} + +func IsActiveOrder(o Order) bool { + return o.Status == OrderStatusNew || o.Status == OrderStatusPartiallyFilled +} From cb2d9d7eb2432e8ae96105767c3c2a5a887fada5 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Dec 2022 16:14:39 +0800 Subject: [PATCH 04/13] grid2: fix replayOrderHistory logic --- pkg/strategy/grid2/strategy.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index f2e59f639..6ac1eab86 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1042,6 +1042,12 @@ func (s *Strategy) recoverGrid(ctx context.Context, historyService types.Exchang } } + missingPrices := scanMissingPinPrices(orderBook, grid.Pins) + if len(missingPrices) == 0 { + s.logger.Infof("GRID RECOVER: no missing grid prices, stop re-playing order history") + return nil + } + // Note that for MAX Exchange, the order history API only uses fromID parameter to query history order. // The time range does not matter. // TODO: handle context correctly @@ -1083,24 +1089,34 @@ func (s *Strategy) recoverGrid(ctx context.Context, historyService types.Exchang return nil } +// replayOrderHistory queries the closed order history from the API and rebuild the orderbook from the order history. +// startTime, endTime is the time range of the order history. func (s *Strategy) replayOrderHistory(ctx context.Context, grid *Grid, orderBook *bbgo.ActiveOrderBook, historyService types.ExchangeTradeHistoryService, startTime, endTime time.Time, lastOrderID uint64) error { gridPriceMap := buildGridPriceMap(grid) // a simple guard, in reality, this startTime is not possible to exceed the endTime // because the queries closed orders might still in the range. - for startTime.Before(endTime) { + orderIdChanged := true + for startTime.Before(endTime) && orderIdChanged { closedOrders, err := historyService.QueryClosedOrders(ctx, s.Symbol, startTime, endTime, lastOrderID) if err != nil { return err } - // need to prevent infinite loop for: len(closedOrders) == 1 and it's creationTime = startTime - if len(closedOrders) == 0 || len(closedOrders) == 1 && closedOrders[0].CreationTime.Time().Equal(startTime) { + // need to prevent infinite loop for: + // if there is only one order and the order creation time matches our startTime + if len(closedOrders) == 0 || len(closedOrders) == 1 && closedOrders[0].OrderID == lastOrderID { break } // for each closed order, if it's newer than the open order's update time, we will update it. + orderIdChanged = false for _, closedOrder := range closedOrders { + if closedOrder.OrderID > lastOrderID { + lastOrderID = closedOrder.OrderID + orderIdChanged = true + } + // skip orders that are not limit order if closedOrder.Type != types.OrderTypeLimit { continue @@ -1135,12 +1151,6 @@ func (s *Strategy) replayOrderHistory(ctx context.Context, grid *Grid, orderBook } } } - - missingPrices := scanMissingPinPrices(orderBook, grid.Pins) - if len(missingPrices) == 0 { - s.logger.Infof("GRID RECOVER: no missing grid prices, stop re-playing order history") - break - } } return nil From e0daf9904e83068e8cd9dda51e09741b0e58c497 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Dec 2022 17:08:50 +0800 Subject: [PATCH 05/13] grid2: add recover time range rollback --- pkg/strategy/grid2/strategy.go | 42 ++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 6ac1eab86..c322d8963 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -26,6 +26,9 @@ var log = logrus.WithField("strategy", ID) var maxNumberOfOrderTradesQueryTries = 10 +const historyRollbackDuration = 3 * 24 * time.Hour +const historyRollbackOrderIdRange = 1000 + func init() { // Register the pointer of the strategy struct, // so that bbgo knows what struct to be used to unmarshal the configs (YAML or JSON) @@ -1018,14 +1021,13 @@ func (s *Strategy) recoverGrid(ctx context.Context, historyService types.Exchang firstOrderTime = since lastOrderTime = until } + _ = lastOrderTime // for MAX exchange we need the order ID to query the closed order history if oid, ok := findEarliestOrderID(openOrders); ok { lastOrderID = oid } - _ = lastOrderTime - activeOrderBook := s.orderExecutor.ActiveMakerOrders() // Allocate a local order book @@ -1042,17 +1044,39 @@ func (s *Strategy) recoverGrid(ctx context.Context, historyService types.Exchang } } + // if all open orders are the grid orders, then we don't have to recover missingPrices := scanMissingPinPrices(orderBook, grid.Pins) - if len(missingPrices) == 0 { + if numMissing := len(missingPrices); numMissing <= 1 { s.logger.Infof("GRID RECOVER: no missing grid prices, stop re-playing order history") return nil - } + } else { + // Note that for MAX Exchange, the order history API only uses fromID parameter to query history order. + // The time range does not matter. + // TODO: handle context correctly + startTime := firstOrderTime + endTime := now + maxTries := 3 + for maxTries > 0 { + maxTries-- + if err := s.replayOrderHistory(ctx, grid, orderBook, historyService, startTime, endTime, lastOrderID); err != nil { + return err + } - // Note that for MAX Exchange, the order history API only uses fromID parameter to query history order. - // The time range does not matter. - // TODO: handle context correctly - if err := s.replayOrderHistory(ctx, grid, orderBook, historyService, firstOrderTime, now, lastOrderID); err != nil { - return err + // Verify if there are still missing prices + missingPrices = scanMissingPinPrices(orderBook, grid.Pins) + if len(missingPrices) <= 1 { + // skip this order history loop and start recovering + break + } + + // history rollback range + startTime = startTime.Add(-historyRollbackDuration) + if newFromOrderID := lastOrderID - historyRollbackOrderIdRange; newFromOrderID > 1 { + lastOrderID = newFromOrderID + } + + s.logger.Infof("GRID RECOVER: there are still more than two missing orders, rolling back query start time to earlier time point %s, fromID %d", startTime.String(), lastOrderID) + } } debugGrid(grid, orderBook) From 4388bc209bdfe5bff2925b1456677cffaecfdead Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Dec 2022 20:37:53 +0800 Subject: [PATCH 06/13] types: add simple duration type for parsing [0-9]+[wd] --- pkg/types/duration.go | 47 +++++++++++++++++++++++++++++++++++++++++++ pkg/types/market.go | 6 ++++++ 2 files changed, 53 insertions(+) create mode 100644 pkg/types/duration.go diff --git a/pkg/types/duration.go b/pkg/types/duration.go new file mode 100644 index 000000000..9ec93d9bb --- /dev/null +++ b/pkg/types/duration.go @@ -0,0 +1,47 @@ +package types + +import ( + "regexp" + "strconv" + "time" + + "github.com/pkg/errors" +) + +var simpleDurationRegExp = regexp.MustCompile("^(\\d+)[hdw]$") + +var ErrNotSimpleDuration = errors.New("the given input is not simple duration format") + +type SimpleDuration struct { + Num int64 + Unit string + Duration Duration +} + +func ParseSimpleDuration(s string) (*SimpleDuration, error) { + if !simpleDurationRegExp.MatchString(s) { + return nil, errors.Wrapf(ErrNotSimpleDuration, "input %q is not a simple duration", s) + } + + matches := simpleDurationRegExp.FindStringSubmatch(s) + numStr := matches[1] + unit := matches[2] + num, err := strconv.ParseInt(numStr, 10, 64) + if err != nil { + return nil, err + } + + switch unit { + case "d": + d := Duration(time.Duration(num) * 24 * time.Hour) + return &SimpleDuration{num, unit, d}, nil + case "w": + d := Duration(time.Duration(num) * 7 * 24 * time.Hour) + return &SimpleDuration{num, unit, d}, nil + case "h": + d := Duration(time.Duration(num) * time.Hour) + return &SimpleDuration{num, unit, d}, nil + } + + return nil, errors.Wrapf(ErrNotSimpleDuration, "input %q is not a simple duration", s) +} diff --git a/pkg/types/market.go b/pkg/types/market.go index e2b08cd21..e9a9bd6b5 100644 --- a/pkg/types/market.go +++ b/pkg/types/market.go @@ -26,6 +26,12 @@ func (d *Duration) UnmarshalJSON(data []byte) error { switch t := o.(type) { case string: + sd, err := ParseSimpleDuration(t) + if err == nil { + *d = sd.Duration + return nil + } + dd, err := time.ParseDuration(t) if err != nil { return err From d27786d5aed7aa1e1efd9bdbe06e262c971db301 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Dec 2022 20:39:01 +0800 Subject: [PATCH 07/13] types: always use pointer on duration --- pkg/types/market.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/types/market.go b/pkg/types/market.go index e9a9bd6b5..23f3610ca 100644 --- a/pkg/types/market.go +++ b/pkg/types/market.go @@ -13,8 +13,8 @@ import ( type Duration time.Duration -func (d Duration) Duration() time.Duration { - return time.Duration(d) +func (d *Duration) Duration() time.Duration { + return time.Duration(*d) } func (d *Duration) UnmarshalJSON(data []byte) error { From f60b4630c5b0e1d262a2ed1bca6317f15d8edefe Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Dec 2022 20:39:11 +0800 Subject: [PATCH 08/13] grid2: add AutoRange parameter --- pkg/strategy/grid2/strategy.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index c322d8963..972650241 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -65,6 +65,8 @@ type Strategy struct { // GridNum is the grid number, how many orders you want to post on the orderbook. GridNum int64 `json:"gridNumber"` + AutoRange types.Duration `json:"autoRange"` + UpperPrice fixedpoint.Value `json:"upperPrice"` LowerPrice fixedpoint.Value `json:"lowerPrice"` From 579df0cec9ca6f1b79094ed08fc5d628cd047f59 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 25 Dec 2022 16:08:34 +0800 Subject: [PATCH 09/13] types: add simple duration tests --- pkg/strategy/grid2/strategy.go | 2 +- pkg/types/duration.go | 71 +++++++++++++++++++++++++++++++++- pkg/types/duration_test.go | 55 ++++++++++++++++++++++++++ pkg/types/market.go | 47 ---------------------- 4 files changed, 125 insertions(+), 50 deletions(-) create mode 100644 pkg/types/duration_test.go diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 972650241..6dcca83e2 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -65,7 +65,7 @@ type Strategy struct { // GridNum is the grid number, how many orders you want to post on the orderbook. GridNum int64 `json:"gridNumber"` - AutoRange types.Duration `json:"autoRange"` + AutoRange types.SimpleDuration `json:"autoRange"` UpperPrice fixedpoint.Value `json:"upperPrice"` diff --git a/pkg/types/duration.go b/pkg/types/duration.go index 9ec93d9bb..f465f24bd 100644 --- a/pkg/types/duration.go +++ b/pkg/types/duration.go @@ -1,6 +1,8 @@ package types import ( + "encoding/json" + "fmt" "regexp" "strconv" "time" @@ -8,9 +10,9 @@ import ( "github.com/pkg/errors" ) -var simpleDurationRegExp = regexp.MustCompile("^(\\d+)[hdw]$") +var simpleDurationRegExp = regexp.MustCompile("^(\\d+)([hdw])$") -var ErrNotSimpleDuration = errors.New("the given input is not simple duration format") +var ErrNotSimpleDuration = errors.New("the given input is not simple duration format, valid format: [1-9][0-9]*[hdw]") type SimpleDuration struct { Num int64 @@ -18,7 +20,28 @@ type SimpleDuration struct { Duration Duration } +func (d *SimpleDuration) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + + sd, err := ParseSimpleDuration(s) + if err != nil { + return err + } + + if sd != nil { + *d = *sd + } + return nil +} + func ParseSimpleDuration(s string) (*SimpleDuration, error) { + if s == "" { + return nil, nil + } + if !simpleDurationRegExp.MatchString(s) { return nil, errors.Wrapf(ErrNotSimpleDuration, "input %q is not a simple duration", s) } @@ -45,3 +68,47 @@ func ParseSimpleDuration(s string) (*SimpleDuration, error) { return nil, errors.Wrapf(ErrNotSimpleDuration, "input %q is not a simple duration", s) } + +type Duration time.Duration + +func (d *Duration) Duration() time.Duration { + return time.Duration(*d) +} + +func (d *Duration) UnmarshalJSON(data []byte) error { + var o interface{} + + if err := json.Unmarshal(data, &o); err != nil { + return err + } + + switch t := o.(type) { + case string: + sd, err := ParseSimpleDuration(t) + if err == nil { + *d = sd.Duration + return nil + } + + dd, err := time.ParseDuration(t) + if err != nil { + return err + } + + *d = Duration(dd) + + case float64: + *d = Duration(int64(t * float64(time.Second))) + + case int64: + *d = Duration(t * int64(time.Second)) + case int: + *d = Duration(t * int(time.Second)) + + default: + return fmt.Errorf("unsupported type %T value: %v", t, t) + + } + + return nil +} diff --git a/pkg/types/duration_test.go b/pkg/types/duration_test.go new file mode 100644 index 000000000..44a56c80d --- /dev/null +++ b/pkg/types/duration_test.go @@ -0,0 +1,55 @@ +package types + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestParseSimpleDuration(t *testing.T) { + type args struct { + s string + } + tests := []struct { + name string + args args + want *SimpleDuration + wantErr assert.ErrorAssertionFunc + }{ + { + name: "3h", + args: args{ + s: "3h", + }, + want: &SimpleDuration{Num: 3, Unit: "h", Duration: Duration(3 * time.Hour)}, + wantErr: assert.NoError, + }, + { + name: "3d", + args: args{ + s: "3d", + }, + want: &SimpleDuration{Num: 3, Unit: "d", Duration: Duration(3 * 24 * time.Hour)}, + wantErr: assert.NoError, + }, + { + name: "3w", + args: args{ + s: "3w", + }, + want: &SimpleDuration{Num: 3, Unit: "w", Duration: Duration(3 * 7 * 24 * time.Hour)}, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseSimpleDuration(tt.args.s) + if !tt.wantErr(t, err, fmt.Sprintf("ParseSimpleDuration(%v)", tt.args.s)) { + return + } + assert.Equalf(t, tt.want, got, "ParseSimpleDuration(%v)", tt.args.s) + }) + } +} diff --git a/pkg/types/market.go b/pkg/types/market.go index 23f3610ca..6cc9466ff 100644 --- a/pkg/types/market.go +++ b/pkg/types/market.go @@ -1,60 +1,13 @@ package types import ( - "encoding/json" - "fmt" "math" - "time" "github.com/leekchan/accounting" "github.com/c9s/bbgo/pkg/fixedpoint" ) -type Duration time.Duration - -func (d *Duration) Duration() time.Duration { - return time.Duration(*d) -} - -func (d *Duration) UnmarshalJSON(data []byte) error { - var o interface{} - - if err := json.Unmarshal(data, &o); err != nil { - return err - } - - switch t := o.(type) { - case string: - sd, err := ParseSimpleDuration(t) - if err == nil { - *d = sd.Duration - return nil - } - - dd, err := time.ParseDuration(t) - if err != nil { - return err - } - - *d = Duration(dd) - - case float64: - *d = Duration(int64(t * float64(time.Second))) - - case int64: - *d = Duration(t * int64(time.Second)) - case int: - *d = Duration(t * int(time.Second)) - - default: - return fmt.Errorf("unsupported type %T value: %v", t, t) - - } - - return nil -} - type Market struct { Symbol string `json:"symbol"` From 54b4f593ecb5e67a83b4d8865209956432f65718 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 26 Dec 2022 00:29:31 +0800 Subject: [PATCH 10/13] grid2: validate upper price and lower price only when autoRange is not given --- pkg/strategy/grid2/strategy.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 6dcca83e2..549a937dd 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -65,7 +65,7 @@ type Strategy struct { // GridNum is the grid number, how many orders you want to post on the orderbook. GridNum int64 `json:"gridNumber"` - AutoRange types.SimpleDuration `json:"autoRange"` + AutoRange *types.SimpleDuration `json:"autoRange"` UpperPrice fixedpoint.Value `json:"upperPrice"` @@ -139,16 +139,18 @@ func (s *Strategy) ID() string { } func (s *Strategy) Validate() error { - if s.UpperPrice.IsZero() { - return errors.New("upperPrice can not be zero, you forgot to set?") - } + if s.AutoRange == nil { + if s.UpperPrice.IsZero() { + return errors.New("upperPrice can not be zero, you forgot to set?") + } - if s.LowerPrice.IsZero() { - return errors.New("lowerPrice can not be zero, you forgot to set?") - } + if s.LowerPrice.IsZero() { + return errors.New("lowerPrice can not be zero, you forgot to set?") + } - if s.UpperPrice.Compare(s.LowerPrice) <= 0 { - return fmt.Errorf("upperPrice (%s) should not be less than or equal to lowerPrice (%s)", s.UpperPrice.String(), s.LowerPrice.String()) + if s.UpperPrice.Compare(s.LowerPrice) <= 0 { + return fmt.Errorf("upperPrice (%s) should not be less than or equal to lowerPrice (%s)", s.UpperPrice.String(), s.LowerPrice.String()) + } } if s.GridNum == 0 { From 961725f03c8ee03bf35a38137b5a106acb083c78 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 26 Dec 2022 00:56:03 +0800 Subject: [PATCH 11/13] grid2: support autoRange --- pkg/strategy/grid2/strategy.go | 16 +++++++++++++++- pkg/types/duration.go | 20 ++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 549a937dd..bff7447d3 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -172,6 +172,11 @@ func (s *Strategy) Validate() error { func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: types.Interval1m}) + + if s.AutoRange != nil { + interval := s.AutoRange.Interval() + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: interval}) + } } // InstanceID returns the instance identifier from the current grid configuration parameters @@ -1301,7 +1306,16 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. }) s.groupID = util.FNV32(instanceID) - s.logger.Infof("using group id %d from fnv(%s)", s.groupID, instanceID) + + if s.AutoRange != nil { + indicatorSet := session.StandardIndicatorSet(s.Symbol) + interval := s.AutoRange.Interval() + pivotLow := indicatorSet.PivotLow(types.IntervalWindow{Interval: interval, Window: s.AutoRange.Num}) + pivotHigh := indicatorSet.PivotHigh(types.IntervalWindow{Interval: interval, Window: s.AutoRange.Num}) + s.UpperPrice = fixedpoint.NewFromFloat(pivotHigh.Last()) + s.LowerPrice = fixedpoint.NewFromFloat(pivotLow.Last()) + s.logger.Infof("autoRange is enabled, using pivot high %f and pivot low %f", s.UpperPrice.Float64(), s.LowerPrice.Float64()) + } if s.ProfitSpread.Sign() > 0 { s.ProfitSpread = s.Market.TruncatePrice(s.ProfitSpread) diff --git a/pkg/types/duration.go b/pkg/types/duration.go index f465f24bd..881cbe4d6 100644 --- a/pkg/types/duration.go +++ b/pkg/types/duration.go @@ -15,11 +15,27 @@ var simpleDurationRegExp = regexp.MustCompile("^(\\d+)([hdw])$") var ErrNotSimpleDuration = errors.New("the given input is not simple duration format, valid format: [1-9][0-9]*[hdw]") type SimpleDuration struct { - Num int64 + Num int Unit string Duration Duration } +func (d *SimpleDuration) Interval() Interval { + switch d.Unit { + + case "d": + return Interval1d + case "h": + return Interval1h + + case "w": + return Interval1w + + } + + return "" +} + func (d *SimpleDuration) UnmarshalJSON(data []byte) error { var s string if err := json.Unmarshal(data, &s); err != nil { @@ -49,7 +65,7 @@ func ParseSimpleDuration(s string) (*SimpleDuration, error) { matches := simpleDurationRegExp.FindStringSubmatch(s) numStr := matches[1] unit := matches[2] - num, err := strconv.ParseInt(numStr, 10, 64) + num, err := strconv.Atoi(numStr) if err != nil { return nil, err } From 24f0f40fad2e4cb45dbbdf3edb9b16d366086fdd Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 26 Dec 2022 01:00:15 +0800 Subject: [PATCH 12/13] config: add autoRange config doc --- config/grid2.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/grid2.yaml b/config/grid2.yaml index aed8bc28d..18725fe9d 100644 --- a/config/grid2.yaml +++ b/config/grid2.yaml @@ -55,6 +55,15 @@ exchangeStrategies: - on: binance grid2: symbol: BTCUSDT + + ## autoRange can be used to detect a price range from a specific time frame + ## the pivot low / pivot high of the given range will be used for lowerPrice and upperPrice. + ## when autoRange is set, it will override the upperPrice/lowerPrice settings. + ## + ## the valid format is [1-9][hdw] + ## example: "14d" means it will find the highest/lowest price that is higher/lower than left 14d and right 14d. + # autoRange: 14d + lowerPrice: 28_000.0 upperPrice: 50_000.0 From 0a6261b6b99be501afcca27f0f4a861eb0d1d1a0 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 26 Dec 2022 01:04:17 +0800 Subject: [PATCH 13/13] grid2: split more files --- pkg/strategy/grid2/debug.go | 48 ++++++++++++++++++++++ pkg/strategy/grid2/pricemap.go | 16 ++++++++ pkg/strategy/grid2/strategy.go | 74 ---------------------------------- pkg/strategy/grid2/trade.go | 27 +++++++++++++ 4 files changed, 91 insertions(+), 74 deletions(-) create mode 100644 pkg/strategy/grid2/debug.go create mode 100644 pkg/strategy/grid2/pricemap.go create mode 100644 pkg/strategy/grid2/trade.go diff --git a/pkg/strategy/grid2/debug.go b/pkg/strategy/grid2/debug.go new file mode 100644 index 000000000..56fbf92cb --- /dev/null +++ b/pkg/strategy/grid2/debug.go @@ -0,0 +1,48 @@ +package grid2 + +import ( + "fmt" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func debugGrid(grid *Grid, book *bbgo.ActiveOrderBook) { + fmt.Println("================== GRID ORDERS ==================") + + pins := grid.Pins + missingPins := scanMissingPinPrices(book, pins) + missing := len(missingPins) + + for i := len(pins) - 1; i >= 0; i-- { + pin := pins[i] + price := fixedpoint.Value(pin) + + fmt.Printf("%s -> ", price.String()) + + existingOrder := book.Lookup(func(o types.Order) bool { + return o.Price.Eq(price) + }) + + if existingOrder != nil { + fmt.Printf("%s", existingOrder.String()) + + switch existingOrder.Status { + case types.OrderStatusFilled: + fmt.Printf(" | 🔧") + case types.OrderStatusCanceled: + fmt.Printf(" | 🔄") + default: + fmt.Printf(" | ✅") + } + } else { + fmt.Printf("ORDER MISSING ⚠️ ") + if missing == 1 { + fmt.Printf(" COULD BE EMPTY SLOT") + } + } + fmt.Printf("\n") + } + fmt.Println("================== END OF GRID ORDERS ===================") +} diff --git a/pkg/strategy/grid2/pricemap.go b/pkg/strategy/grid2/pricemap.go new file mode 100644 index 000000000..04ae59711 --- /dev/null +++ b/pkg/strategy/grid2/pricemap.go @@ -0,0 +1,16 @@ +package grid2 + +import "github.com/c9s/bbgo/pkg/fixedpoint" + +type PriceMap map[string]fixedpoint.Value + +func buildGridPriceMap(grid *Grid) PriceMap { + // Add all open orders to the local order book + gridPriceMap := make(PriceMap) + for _, pin := range grid.Pins { + price := fixedpoint.Value(pin) + gridPriceMap[price.String()] = price + } + + return gridPriceMap +} diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index bff7447d3..5403b453c 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -20,8 +20,6 @@ const ID = "grid2" const orderTag = "grid2" -type PriceMap map[string]fixedpoint.Value - var log = logrus.WithField("strategy", ID) var maxNumberOfOrderTradesQueryTries = 10 @@ -247,27 +245,6 @@ func (s *Strategy) calculateProfit(o types.Order, buyPrice, buyQuantity fixedpoi return profit } -// collectTradeFee collects the fee from the given trade slice -func collectTradeFee(trades []types.Trade) map[string]fixedpoint.Value { - fees := make(map[string]fixedpoint.Value) - for _, t := range trades { - if fee, ok := fees[t.FeeCurrency]; ok { - fees[t.FeeCurrency] = fee.Add(t.Fee) - } else { - fees[t.FeeCurrency] = t.Fee - } - } - return fees -} - -func aggregateTradesQuantity(trades []types.Trade) fixedpoint.Value { - tq := fixedpoint.Zero - for _, t := range trades { - tq = tq.Add(t.Quantity) - } - return tq -} - func (s *Strategy) verifyOrderTrades(o types.Order, trades []types.Trade) bool { tq := aggregateTradesQuantity(trades) @@ -757,7 +734,6 @@ func (s *Strategy) newGrid() *Grid { // openGrid // 1) if quantity or amount is set, we should use quantity/amount directly instead of using investment amount to calculate. // 2) if baseInvestment, quoteInvestment is set, then we should calculate the quantity from the given base investment and quote investment. -// TODO: fix sell order placement for profitSpread func (s *Strategy) openGrid(ctx context.Context, session *bbgo.ExchangeSession) error { // grid object guard if s.grid != nil { @@ -1005,17 +981,6 @@ func (s *Strategy) checkMinimalQuoteInvestment() error { return nil } -func buildGridPriceMap(grid *Grid) PriceMap { - // Add all open orders to the local order book - gridPriceMap := make(PriceMap) - for _, pin := range grid.Pins { - price := fixedpoint.Value(pin) - gridPriceMap[price.String()] = price - } - - return gridPriceMap -} - func (s *Strategy) recoverGrid(ctx context.Context, historyService types.ExchangeTradeHistoryService, openOrders []types.Order) error { grid := s.newGrid() @@ -1199,45 +1164,6 @@ func isCompleteGridOrderBook(orderBook *bbgo.ActiveOrderBook, gridNum int64) boo return false } -func debugGrid(grid *Grid, book *bbgo.ActiveOrderBook) { - fmt.Println("================== GRID ORDERS ==================") - - pins := grid.Pins - missingPins := scanMissingPinPrices(book, pins) - missing := len(missingPins) - - for i := len(pins) - 1; i >= 0; i-- { - pin := pins[i] - price := fixedpoint.Value(pin) - - fmt.Printf("%s -> ", price.String()) - - existingOrder := book.Lookup(func(o types.Order) bool { - return o.Price.Eq(price) - }) - - if existingOrder != nil { - fmt.Printf("%s", existingOrder.String()) - - switch existingOrder.Status { - case types.OrderStatusFilled: - fmt.Printf(" | 🔧") - case types.OrderStatusCanceled: - fmt.Printf(" | 🔄") - default: - fmt.Printf(" | ✅") - } - } else { - fmt.Printf("ORDER MISSING ⚠️ ") - if missing == 1 { - fmt.Printf(" COULD BE EMPTY SLOT") - } - } - fmt.Printf("\n") - } - fmt.Println("================== END OF GRID ORDERS ===================") -} - func findEarliestOrderID(orders []types.Order) (uint64, bool) { if len(orders) == 0 { return 0, false diff --git a/pkg/strategy/grid2/trade.go b/pkg/strategy/grid2/trade.go new file mode 100644 index 000000000..381744ccd --- /dev/null +++ b/pkg/strategy/grid2/trade.go @@ -0,0 +1,27 @@ +package grid2 + +import ( + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +// collectTradeFee collects the fee from the given trade slice +func collectTradeFee(trades []types.Trade) map[string]fixedpoint.Value { + fees := make(map[string]fixedpoint.Value) + for _, t := range trades { + if fee, ok := fees[t.FeeCurrency]; ok { + fees[t.FeeCurrency] = fee.Add(t.Fee) + } else { + fees[t.FeeCurrency] = t.Fee + } + } + return fees +} + +func aggregateTradesQuantity(trades []types.Trade) fixedpoint.Value { + tq := fixedpoint.Zero + for _, t := range trades { + tq = tq.Add(t.Quantity) + } + return tq +}