From c9fca567235c6f3a4f226a1ecc438aa3dd74fdcb Mon Sep 17 00:00:00 2001
From: chiahung <s199180626@gmail.com>
Date: Fri, 20 Oct 2023 15:17:31 +0800
Subject: [PATCH 01/42] MINOR: remove profit entries from profit stats

---
 pkg/strategy/grid2/profit_stats.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/pkg/strategy/grid2/profit_stats.go b/pkg/strategy/grid2/profit_stats.go
index 5f47effc6..cd8367c23 100644
--- a/pkg/strategy/grid2/profit_stats.go
+++ b/pkg/strategy/grid2/profit_stats.go
@@ -22,7 +22,6 @@ type GridProfitStats struct {
 	TotalFee         map[string]fixedpoint.Value `json:"totalFee,omitempty"`
 	Volume           fixedpoint.Value            `json:"volume,omitempty"`
 	Market           types.Market                `json:"market,omitempty"`
-	ProfitEntries    []*GridProfit               `json:"profitEntries,omitempty"`
 	Since            *time.Time                  `json:"since,omitempty"`
 	InitialOrderID   uint64                      `json:"initialOrderID"`
 }
@@ -38,7 +37,6 @@ func newGridProfitStats(market types.Market) *GridProfitStats {
 		TotalFee:         make(map[string]fixedpoint.Value),
 		Volume:           fixedpoint.Zero,
 		Market:           market,
-		ProfitEntries:    nil,
 	}
 }
 
@@ -69,8 +67,6 @@ func (s *GridProfitStats) AddProfit(profit *GridProfit) {
 	case s.Market.BaseCurrency:
 		s.TotalBaseProfit = s.TotalBaseProfit.Add(profit.Profit)
 	}
-
-	s.ProfitEntries = append(s.ProfitEntries, profit)
 }
 
 func (s *GridProfitStats) SlackAttachment() slack.Attachment {

From e9078a71c8c62e4c7f568ed83d37f8c564123129 Mon Sep 17 00:00:00 2001
From: chiahung <s199180626@gmail.com>
Date: Fri, 20 Oct 2023 16:23:18 +0800
Subject: [PATCH 02/42] FEATURE: twin orderbook

---
 pkg/strategy/grid2/twin_order.go      | 125 ++++++++++++++++++++++++++
 pkg/strategy/grid2/twin_order_test.go |  68 ++++++++++++++
 2 files changed, 193 insertions(+)
 create mode 100644 pkg/strategy/grid2/twin_order_test.go

diff --git a/pkg/strategy/grid2/twin_order.go b/pkg/strategy/grid2/twin_order.go
index adeeb5263..9b9cf7e16 100644
--- a/pkg/strategy/grid2/twin_order.go
+++ b/pkg/strategy/grid2/twin_order.go
@@ -111,3 +111,128 @@ func (m TwinOrderMap) String() string {
 	sb.WriteString("================== END OF PIN ORDER MAP ==================\n")
 	return sb.String()
 }
+
+type TwinOrderBook struct {
+	// sort in asc order
+	pins []fixedpoint.Value
+
+	// pin index, use to find the next or last pin in desc order
+	pinIdx map[fixedpoint.Value]int
+
+	// orderbook
+	m map[fixedpoint.Value]*TwinOrder
+
+	size int
+}
+
+func NewTwinOrderBook(pins []Pin) *TwinOrderBook {
+	var v []fixedpoint.Value
+	for _, pin := range pins {
+		v = append(v, fixedpoint.Value(pin))
+	}
+
+	// sort it in asc order
+	sort.Slice(v, func(i, j int) bool {
+		return v[j].Compare(v[i]) > 0
+	})
+
+	pinIdx := make(map[fixedpoint.Value]int)
+	m := make(map[fixedpoint.Value]*TwinOrder)
+	for i, pin := range v {
+		m[pin] = &TwinOrder{}
+		pinIdx[pin] = i
+	}
+
+	ob := TwinOrderBook{
+		pins:   v,
+		pinIdx: pinIdx,
+		m:      m,
+		size:   0,
+	}
+
+	return &ob
+}
+
+func (book *TwinOrderBook) String() string {
+	var sb strings.Builder
+
+	sb.WriteString("================== TWIN ORDERBOOK ==================\n")
+	for _, pin := range book.pins {
+		twin := book.m[fixedpoint.Value(pin)]
+		twinOrder := twin.GetOrder()
+		sb.WriteString(fmt.Sprintf("-> %8s) %s\n", pin, twinOrder.String()))
+	}
+	sb.WriteString("================== END OF TWINORDERBOOK ==================\n")
+	return sb.String()
+}
+
+func (book *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, error) {
+	idx, exist := book.pinIdx[order.Price]
+	if !exist {
+		return fixedpoint.Zero, fmt.Errorf("the order's (%d) price (%s) is not in pins", order.OrderID, order.Price)
+	}
+
+	if order.Side == types.SideTypeBuy {
+		idx++
+		if idx >= len(book.pins) {
+			return fixedpoint.Zero, fmt.Errorf("this order's twin order price is not in pins, %+v", order)
+		}
+	} else if order.Side == types.SideTypeSell {
+		if idx == 0 {
+			return fixedpoint.Zero, fmt.Errorf("this order's twin order price is at zero index, %+v", order)
+		}
+		// do nothing
+	} else {
+		// should not happen
+		return fixedpoint.Zero, fmt.Errorf("the order's (%d) side (%s) is not supported", order.OrderID, order.Side)
+	}
+
+	return book.pins[idx], nil
+}
+
+func (book *TwinOrderBook) AddOrder(order types.Order) error {
+	pin, err := book.GetTwinOrderPin(order)
+	if err != nil {
+		return err
+	}
+
+	twinOrder, exist := book.m[pin]
+	if !exist {
+		// should not happen
+		return fmt.Errorf("no any empty twin order at pins, should not happen, check it")
+	}
+
+	if !twinOrder.Exist() {
+		book.size++
+	}
+	twinOrder.SetOrder(order)
+
+	return nil
+}
+
+func (book *TwinOrderBook) GetTwinOrder(pin fixedpoint.Value) *TwinOrder {
+	return book.m[pin]
+}
+
+func (book *TwinOrderBook) AddTwinOrder(pin fixedpoint.Value, order *TwinOrder) {
+	book.m[pin] = order
+}
+
+func (book *TwinOrderBook) Size() int {
+	return book.size
+}
+
+func (book *TwinOrderBook) EmptyTwinOrderSize() int {
+	return len(book.pins) - 1 - book.size
+}
+
+func (book *TwinOrderBook) SyncOrderMap() *types.SyncOrderMap {
+	orderMap := types.NewSyncOrderMap()
+	for _, twin := range book.m {
+		if twin.Exist() {
+			orderMap.Add(twin.GetOrder())
+		}
+	}
+
+	return orderMap
+}
diff --git a/pkg/strategy/grid2/twin_order_test.go b/pkg/strategy/grid2/twin_order_test.go
new file mode 100644
index 000000000..eb75e93a1
--- /dev/null
+++ b/pkg/strategy/grid2/twin_order_test.go
@@ -0,0 +1,68 @@
+package grid2
+
+import (
+	"testing"
+
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTwinOrderBook(t *testing.T) {
+	assert := assert.New(t)
+	pins := []Pin{
+		Pin(fixedpoint.NewFromInt(3)),
+		Pin(fixedpoint.NewFromInt(4)),
+		Pin(fixedpoint.NewFromInt(1)),
+		Pin(fixedpoint.NewFromInt(5)),
+		Pin(fixedpoint.NewFromInt(2)),
+	}
+
+	book := NewTwinOrderBook(pins)
+	assert.Equal(0, book.Size())
+	assert.Equal(4, book.EmptyTwinOrderSize())
+	for _, pin := range pins {
+		twinOrder := book.GetTwinOrder(fixedpoint.Value(pin))
+		if !assert.NotNil(twinOrder) {
+			continue
+		}
+
+		assert.False(twinOrder.Exist())
+	}
+
+	orders := []types.Order{
+		{
+			OrderID: 1,
+			SubmitOrder: types.SubmitOrder{
+				Price: fixedpoint.NewFromInt(2),
+				Side:  types.SideTypeBuy,
+			},
+		},
+		{
+			OrderID: 2,
+			SubmitOrder: types.SubmitOrder{
+				Price: fixedpoint.NewFromInt(4),
+				Side:  types.SideTypeSell,
+			},
+		},
+	}
+
+	for _, order := range orders {
+		assert.NoError(book.AddOrder(order))
+	}
+	assert.Equal(2, book.Size())
+	assert.Equal(2, book.EmptyTwinOrderSize())
+
+	for _, order := range orders {
+		pin, err := book.GetTwinOrderPin(order)
+		if !assert.NoError(err) {
+			continue
+		}
+		twinOrder := book.GetTwinOrder(pin)
+		if !assert.True(twinOrder.Exist()) {
+			continue
+		}
+
+		assert.Equal(order.OrderID, twinOrder.GetOrder().OrderID)
+	}
+}

From 2ef86e4e7194ee631b5f896f702b4bd56e735db9 Mon Sep 17 00:00:00 2001
From: Samiksha Mishra <38784342+mishrasamiksha@users.noreply.github.com>
Date: Sat, 21 Oct 2023 17:51:33 +0530
Subject: [PATCH 03/42] Update README.md

Corrected typo error
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b947f5aca..eb0011a00 100644
--- a/README.md
+++ b/README.md
@@ -107,7 +107,7 @@ the implementation.
 | irr         | this strategy opens the position based on the predicated return rate                                                                    | long/short |                  |
 | bollmaker   | this strategy holds a long-term long/short position, places maker orders on both side, uses bollinger band to control the position size | maker      |                  |
 | wall        | this strategy creates wall (large amount order) on the order book                                                                       | maker      | no               |
-| scmaker     | this market making strategy is desgiend for stable coin markets, like USDC/USDT                                                         | maker      |                  |
+| scmaker     | this market making strategy is designed for stable coin markets, like USDC/USDT                                                         | maker      |                  |
 | drift       |                                                                                                                                         | long/short |                  |
 | rsicross    | this strategy opens a long position when the fast rsi cross over the slow rsi, this is a demo strategy for using the v2 indicator       | long/short |                  |
 | marketcap   | this strategy implements a strategy that rebalances the portfolio based on the market capitalization                                    | rebalance  | no               |

From da5f8f57f36155a4005d406c4a2c69c702c0c9be Mon Sep 17 00:00:00 2001
From: Sakshi Umredkar <128240967+saakshii12@users.noreply.github.com>
Date: Sun, 22 Oct 2023 17:36:08 +0530
Subject: [PATCH 04/42] Fixed a typo in README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b947f5aca..7f84135ae 100644
--- a/README.md
+++ b/README.md
@@ -488,7 +488,7 @@ See also:
 
 ## Command Usages
 
-### Submitting Orders to a specific exchagne session
+### Submitting Orders to a specific exchange session
 
 ```shell
 bbgo submit-order --session=okex --symbol=OKBUSDT --side=buy --price=10.0 --quantity=1

From 3150f6b3f57504486560632b58a864584b3658b0 Mon Sep 17 00:00:00 2001
From: chiahung <s199180626@gmail.com>
Date: Mon, 23 Oct 2023 13:00:17 +0800
Subject: [PATCH 05/42] fix

---
 pkg/strategy/grid2/twin_order.go      | 48 +++++++++++++--------------
 pkg/strategy/grid2/twin_order_test.go |  2 +-
 2 files changed, 24 insertions(+), 26 deletions(-)

diff --git a/pkg/strategy/grid2/twin_order.go b/pkg/strategy/grid2/twin_order.go
index 9b9cf7e16..bfdf57721 100644
--- a/pkg/strategy/grid2/twin_order.go
+++ b/pkg/strategy/grid2/twin_order.go
@@ -125,7 +125,7 @@ type TwinOrderBook struct {
 	size int
 }
 
-func NewTwinOrderBook(pins []Pin) *TwinOrderBook {
+func newTwinOrderBook(pins []Pin) *TwinOrderBook {
 	var v []fixedpoint.Value
 	for _, pin := range pins {
 		v = append(v, fixedpoint.Value(pin))
@@ -143,22 +143,20 @@ func NewTwinOrderBook(pins []Pin) *TwinOrderBook {
 		pinIdx[pin] = i
 	}
 
-	ob := TwinOrderBook{
+	return &TwinOrderBook{
 		pins:   v,
 		pinIdx: pinIdx,
 		m:      m,
 		size:   0,
 	}
-
-	return &ob
 }
 
-func (book *TwinOrderBook) String() string {
+func (b *TwinOrderBook) String() string {
 	var sb strings.Builder
 
 	sb.WriteString("================== TWIN ORDERBOOK ==================\n")
-	for _, pin := range book.pins {
-		twin := book.m[fixedpoint.Value(pin)]
+	for _, pin := range b.pins {
+		twin := b.m[fixedpoint.Value(pin)]
 		twinOrder := twin.GetOrder()
 		sb.WriteString(fmt.Sprintf("-> %8s) %s\n", pin, twinOrder.String()))
 	}
@@ -166,15 +164,15 @@ func (book *TwinOrderBook) String() string {
 	return sb.String()
 }
 
-func (book *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, error) {
-	idx, exist := book.pinIdx[order.Price]
+func (b *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, error) {
+	idx, exist := b.pinIdx[order.Price]
 	if !exist {
 		return fixedpoint.Zero, fmt.Errorf("the order's (%d) price (%s) is not in pins", order.OrderID, order.Price)
 	}
 
 	if order.Side == types.SideTypeBuy {
 		idx++
-		if idx >= len(book.pins) {
+		if idx >= len(b.pins) {
 			return fixedpoint.Zero, fmt.Errorf("this order's twin order price is not in pins, %+v", order)
 		}
 	} else if order.Side == types.SideTypeSell {
@@ -187,48 +185,48 @@ func (book *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value,
 		return fixedpoint.Zero, fmt.Errorf("the order's (%d) side (%s) is not supported", order.OrderID, order.Side)
 	}
 
-	return book.pins[idx], nil
+	return b.pins[idx], nil
 }
 
-func (book *TwinOrderBook) AddOrder(order types.Order) error {
-	pin, err := book.GetTwinOrderPin(order)
+func (b *TwinOrderBook) AddOrder(order types.Order) error {
+	pin, err := b.GetTwinOrderPin(order)
 	if err != nil {
 		return err
 	}
 
-	twinOrder, exist := book.m[pin]
+	twinOrder, exist := b.m[pin]
 	if !exist {
 		// should not happen
 		return fmt.Errorf("no any empty twin order at pins, should not happen, check it")
 	}
 
 	if !twinOrder.Exist() {
-		book.size++
+		b.size++
 	}
 	twinOrder.SetOrder(order)
 
 	return nil
 }
 
-func (book *TwinOrderBook) GetTwinOrder(pin fixedpoint.Value) *TwinOrder {
-	return book.m[pin]
+func (b *TwinOrderBook) GetTwinOrder(pin fixedpoint.Value) *TwinOrder {
+	return b.m[pin]
 }
 
-func (book *TwinOrderBook) AddTwinOrder(pin fixedpoint.Value, order *TwinOrder) {
-	book.m[pin] = order
+func (b *TwinOrderBook) AddTwinOrder(pin fixedpoint.Value, order *TwinOrder) {
+	b.m[pin] = order
 }
 
-func (book *TwinOrderBook) Size() int {
-	return book.size
+func (b *TwinOrderBook) Size() int {
+	return b.size
 }
 
-func (book *TwinOrderBook) EmptyTwinOrderSize() int {
-	return len(book.pins) - 1 - book.size
+func (b *TwinOrderBook) EmptyTwinOrderSize() int {
+	return len(b.pins) - 1 - b.size
 }
 
-func (book *TwinOrderBook) SyncOrderMap() *types.SyncOrderMap {
+func (b *TwinOrderBook) SyncOrderMap() *types.SyncOrderMap {
 	orderMap := types.NewSyncOrderMap()
-	for _, twin := range book.m {
+	for _, twin := range b.m {
 		if twin.Exist() {
 			orderMap.Add(twin.GetOrder())
 		}
diff --git a/pkg/strategy/grid2/twin_order_test.go b/pkg/strategy/grid2/twin_order_test.go
index eb75e93a1..d6204ee94 100644
--- a/pkg/strategy/grid2/twin_order_test.go
+++ b/pkg/strategy/grid2/twin_order_test.go
@@ -18,7 +18,7 @@ func TestTwinOrderBook(t *testing.T) {
 		Pin(fixedpoint.NewFromInt(2)),
 	}
 
-	book := NewTwinOrderBook(pins)
+	book := newTwinOrderBook(pins)
 	assert.Equal(0, book.Size())
 	assert.Equal(4, book.EmptyTwinOrderSize())
 	for _, pin := range pins {

From 6b5dd653aa69aabfee67d72df12c76256e369d17 Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Mon, 23 Oct 2023 14:38:56 +0800
Subject: [PATCH 06/42] go: update requestgen to v1.3.5

---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 4ebe211fa..91eca5f8b 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/Masterminds/squirrel v1.5.3
 	github.com/adshao/go-binance/v2 v2.4.2
 	github.com/c-bata/goptuna v0.8.1
-	github.com/c9s/requestgen v1.3.4
+	github.com/c9s/requestgen v1.3.5
 	github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b
 	github.com/cenkalti/backoff/v4 v4.2.0
 	github.com/cheggaaa/pb/v3 v3.0.8
diff --git a/go.sum b/go.sum
index 448c0a214..710eba662 100644
--- a/go.sum
+++ b/go.sum
@@ -84,6 +84,8 @@ github.com/c-bata/goptuna v0.8.1 h1:25+n1MLv0yvCsD56xv4nqIus3oLHL9GuPAZDLIqmX1U=
 github.com/c-bata/goptuna v0.8.1/go.mod h1:knmS8+Iyq5PPy1YUeIEq0pMFR4Y6x7z/CySc9HlZTCY=
 github.com/c9s/requestgen v1.3.4 h1:kK2rIO3OAt9JoY5gT0OSkSpq0dy/+JeuI22FwSKpUrY=
 github.com/c9s/requestgen v1.3.4/go.mod h1:wp4saiPdh0zLF5AkopGCqPQfy9Q5xvRh+TQBOA1l1r4=
+github.com/c9s/requestgen v1.3.5 h1:iGYAP0rWQW3JOo+Z3S0SoenSt581IQ9mupJxRFCrCJs=
+github.com/c9s/requestgen v1.3.5/go.mod h1:QwkZudcv84kJ8g9+E0RDTj+13btFXbTvv2aI+zbuLbc=
 github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b h1:wT8c03PHLv7+nZUIGqxAzRvIfYHNxMCNVWwvdGkOXTs=
 github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b/go.mod h1:EKObf66Cp7erWxym2de+07qNN5T1N9PXxHdh97N44EQ=
 github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=

From c977b8e295cf85a190df22bd72719447280f27e9 Mon Sep 17 00:00:00 2001
From: chiahung <s199180626@gmail.com>
Date: Mon, 23 Oct 2023 17:42:39 +0800
Subject: [PATCH 07/42] add lock to protect twin orderbook and add more
 comments

---
 pkg/strategy/grid2/twin_order.go | 42 ++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/pkg/strategy/grid2/twin_order.go b/pkg/strategy/grid2/twin_order.go
index bfdf57721..ccc9bfa4f 100644
--- a/pkg/strategy/grid2/twin_order.go
+++ b/pkg/strategy/grid2/twin_order.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/types"
@@ -112,7 +113,20 @@ func (m TwinOrderMap) String() string {
 	return sb.String()
 }
 
+// TwinOrderBook is to verify grid
+// For grid trading, there are twin orders between a grid
+// e.g. 100, 200, 300, 400, 500
+//      BUY 100 and SELL 200 are a twin.
+//      BUY 200 and SELL 300 are a twin.
+// Because they can't be placed on orderbook at the same time.
+// We use sell price to be the twin orderbook's key
+// New the twin orderbook with pins, and it will sort the pins in asc order.
+// There must be a non nil TwinOrder on the every pin (except the first one).
+// But the TwinOrder.Exist() may be false. It means there is no twin order on this grid
 type TwinOrderBook struct {
+	// used to protect orderbook update
+	mu sync.Mutex
+
 	// sort in asc order
 	pins []fixedpoint.Value
 
@@ -122,6 +136,7 @@ type TwinOrderBook struct {
 	// orderbook
 	m map[fixedpoint.Value]*TwinOrder
 
+	// size is the amount on twin orderbook
 	size int
 }
 
@@ -171,6 +186,17 @@ func (b *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, er
 	}
 
 	if order.Side == types.SideTypeBuy {
+		// we use sell price as twin orderbook's key, so if the order's side is buy.
+		// we need to find its next price on grid.
+		// e.g.
+		// BUY 100 <- twin -> SELL 200
+		// BUY 200 <- twin -> SELL 300
+		// BUY 300 <- twin -> SELL 400
+		// BUY 400 <- twin -> SELL 500
+		// if the order is BUY 100, we need to find its twin order's price to be the twin orderbook's key
+		// so we plus 1 here and use sorted pins to find the next price (200)
+		// there must no BUY 500 in the grid, so we need to make sure the idx should always not over the len(pins)
+		// also, there must no SELL 100 in the grid, so we need to make sure the idx should always not be 0
 		idx++
 		if idx >= len(b.pins) {
 			return fixedpoint.Zero, fmt.Errorf("this order's twin order price is not in pins, %+v", order)
@@ -189,20 +215,30 @@ func (b *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, er
 }
 
 func (b *TwinOrderBook) AddOrder(order types.Order) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
 	pin, err := b.GetTwinOrderPin(order)
 	if err != nil {
 		return err
 	}
 
+	// At all the pins, we already create the empty TwinOrder{}
+	// As a result,if the exist is false, it means the pin is not in the twin orderbook.
+	// That's invalid pin, or we have something wrong when new TwinOrderBook
 	twinOrder, exist := b.m[pin]
 	if !exist {
 		// should not happen
 		return fmt.Errorf("no any empty twin order at pins, should not happen, check it")
 	}
 
+	// Exist == false means there is no twin order on this pin
 	if !twinOrder.Exist() {
 		b.size++
 	}
+	if b.size >= len(b.pins) {
+		return fmt.Errorf("the maximum size of twin orderbook is len(pins) - 1, need to check it")
+	}
 	twinOrder.SetOrder(order)
 
 	return nil
@@ -213,14 +249,20 @@ func (b *TwinOrderBook) GetTwinOrder(pin fixedpoint.Value) *TwinOrder {
 }
 
 func (b *TwinOrderBook) AddTwinOrder(pin fixedpoint.Value, order *TwinOrder) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
 	b.m[pin] = order
 }
 
+// Size is the valid twin order on grid.
 func (b *TwinOrderBook) Size() int {
 	return b.size
 }
 
+// EmptyTwinOrderSize is the amount of grid there is no twin order on it.
 func (b *TwinOrderBook) EmptyTwinOrderSize() int {
+	// for grid, there is only pins - 1 order on the grid, so we need to minus 1.
 	return len(b.pins) - 1 - b.size
 }
 

From 3710c33670509d0dfcda6309b6a9aec3907bcfcc Mon Sep 17 00:00:00 2001
From: chiahung <s199180626@gmail.com>
Date: Mon, 23 Oct 2023 13:17:20 +0800
Subject: [PATCH 08/42] REFACTOR: rename file and variable

---
 pkg/strategy/grid2/active_order_recover.go    | 12 +--
 .../grid2/active_order_recover_test.go        | 74 +++++++++++++++++++
 .../grid2/{recover.go => grid_recover.go}     |  0
 .../{recover_test.go => grid_recover_test.go} |  0
 pkg/strategy/grid2/strategy.go                |  2 +-
 5 files changed, 81 insertions(+), 7 deletions(-)
 rename pkg/strategy/grid2/{recover.go => grid_recover.go} (100%)
 rename pkg/strategy/grid2/{recover_test.go => grid_recover_test.go} (100%)

diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go
index e93e722f0..2ffdbb43c 100644
--- a/pkg/strategy/grid2/active_order_recover.go
+++ b/pkg/strategy/grid2/active_order_recover.go
@@ -23,21 +23,21 @@ type SyncActiveOrdersOpts struct {
 	exchange          types.Exchange
 }
 
-func (s *Strategy) initializeRecoverCh() bool {
+func (s *Strategy) initializeRecoverC() bool {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
 	isInitialize := false
 
-	if s.activeOrdersRecoverC == nil {
+	if s.recoverC == nil {
 		s.logger.Info("initializing recover channel")
-		s.activeOrdersRecoverC = make(chan struct{}, 1)
+		s.recoverC = make(chan struct{}, 1)
 	} else {
 		s.logger.Info("recover channel is already initialized, trigger active orders recover")
 		isInitialize = true
 
 		select {
-		case s.activeOrdersRecoverC <- struct{}{}:
+		case s.recoverC <- struct{}{}:
 			s.logger.Info("trigger active orders recover")
 		default:
 			s.logger.Info("activeOrdersRecoverC is full")
@@ -49,7 +49,7 @@ func (s *Strategy) initializeRecoverCh() bool {
 
 func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) {
 	// every time we activeOrdersRecoverC receive signal, do active orders recover
-	if isInitialize := s.initializeRecoverCh(); isInitialize {
+	if isInitialize := s.initializeRecoverC(); isInitialize {
 		return
 	}
 
@@ -78,7 +78,7 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) {
 				log.WithError(err).Errorf("unable to sync active orders")
 			}
 
-		case <-s.activeOrdersRecoverC:
+		case <-s.recoverC:
 			if err := syncActiveOrders(ctx, opts); err != nil {
 				log.WithError(err).Errorf("unable to sync active orders")
 			}
diff --git a/pkg/strategy/grid2/active_order_recover_test.go b/pkg/strategy/grid2/active_order_recover_test.go
index dffdccc38..5a72c05c0 100644
--- a/pkg/strategy/grid2/active_order_recover_test.go
+++ b/pkg/strategy/grid2/active_order_recover_test.go
@@ -174,3 +174,77 @@ func TestSyncActiveOrders(t *testing.T) {
 		assert.Equal(types.OrderStatusNew, activeOrders[0].Status)
 	})
 }
+
+func TestSyncActiveOrder(t *testing.T) {
+	assert := assert.New(t)
+
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	symbol := "ETHUSDT"
+
+	t.Run("sync filled order in active orderbook, active orderbook should remove this order", func(t *testing.T) {
+		mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl)
+		activeOrderbook := bbgo.NewActiveOrderBook(symbol)
+
+		order := types.Order{
+			OrderID: 1,
+			Status:  types.OrderStatusNew,
+			SubmitOrder: types.SubmitOrder{
+				Symbol: symbol,
+			},
+		}
+		activeOrderbook.Add(order)
+
+		updatedOrder := order
+		updatedOrder.Status = types.OrderStatusFilled
+
+		mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{
+			Symbol:  symbol,
+			OrderID: strconv.FormatUint(order.OrderID, 10),
+		}).Return(&updatedOrder, nil)
+
+		if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) {
+			return
+		}
+
+		// verify active orderbook
+		activeOrders := activeOrderbook.Orders()
+		assert.Equal(0, len(activeOrders))
+	})
+
+	t.Run("sync partial-filled order in active orderbook, active orderbook should still keep this order", func(t *testing.T) {
+		mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl)
+		activeOrderbook := bbgo.NewActiveOrderBook(symbol)
+
+		order := types.Order{
+			OrderID: 1,
+			Status:  types.OrderStatusNew,
+			SubmitOrder: types.SubmitOrder{
+				Symbol: symbol,
+			},
+		}
+		activeOrderbook.Add(order)
+
+		updatedOrder := order
+		updatedOrder.Status = types.OrderStatusPartiallyFilled
+
+		mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{
+			Symbol:  symbol,
+			OrderID: strconv.FormatUint(order.OrderID, 10),
+		}).Return(&updatedOrder, nil)
+
+		if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) {
+			return
+		}
+
+		// verify active orderbook
+		activeOrders := activeOrderbook.Orders()
+		assert.Equal(1, len(activeOrders))
+		assert.Equal(order.OrderID, activeOrders[0].OrderID)
+		assert.Equal(updatedOrder.Status, activeOrders[0].Status)
+	})
+}
diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/grid_recover.go
similarity index 100%
rename from pkg/strategy/grid2/recover.go
rename to pkg/strategy/grid2/grid_recover.go
diff --git a/pkg/strategy/grid2/recover_test.go b/pkg/strategy/grid2/grid_recover_test.go
similarity index 100%
rename from pkg/strategy/grid2/recover_test.go
rename to pkg/strategy/grid2/grid_recover_test.go
diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go
index 736b1ad67..ce3f77dc2 100644
--- a/pkg/strategy/grid2/strategy.go
+++ b/pkg/strategy/grid2/strategy.go
@@ -204,7 +204,7 @@ type Strategy struct {
 	tradingCtx, writeCtx context.Context
 	cancelWrite          context.CancelFunc
 
-	activeOrdersRecoverC chan struct{}
+	recoverC chan struct{}
 
 	// this ensures that bbgo.Sync to lock the object
 	sync.Mutex

From a9d9ef379268f6f1a841d3626a73d89612097276 Mon Sep 17 00:00:00 2001
From: c9s <yoanlin93@gmail.com>
Date: Tue, 24 Oct 2023 13:44:25 +0800
Subject: [PATCH 09/42] Add AddSubscriber method on Float64Series

---
 pkg/types/series_float64.go | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/pkg/types/series_float64.go b/pkg/types/series_float64.go
index f12e05d58..8c07b4a13 100644
--- a/pkg/types/series_float64.go
+++ b/pkg/types/series_float64.go
@@ -46,6 +46,21 @@ func (f *Float64Series) Subscribe(source Float64Source, c func(x float64)) {
 	}
 }
 
+// AddSubscriber adds the subscriber function and push historical data to the subscriber
+func (f *Float64Series) AddSubscriber(fn func(v float64)) {
+	f.OnUpdate(fn)
+
+	if f.Length() == 0 {
+		return
+	}
+
+	// push historical values to the subscriber
+	for _, vv := range f.Slice {
+		fn(vv)
+	}
+}
+
+
 // Bind binds the source event to the target (Float64Calculator)
 // A Float64Calculator should be able to calculate the float64 result from a single float64 argument input
 func (f *Float64Series) Bind(source Float64Source, target Float64Calculator) {

From 4c1654652eb20eec8237e51b0c7c92fc3e921524 Mon Sep 17 00:00:00 2001
From: c9s <yoanlin93@gmail.com>
Date: Tue, 24 Oct 2023 13:44:49 +0800
Subject: [PATCH 10/42] indicator: remove unnecessary zero value push

---
 pkg/indicator/v2/rma.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/pkg/indicator/v2/rma.go b/pkg/indicator/v2/rma.go
index 1aa08ccdc..4a6c8b327 100644
--- a/pkg/indicator/v2/rma.go
+++ b/pkg/indicator/v2/rma.go
@@ -48,11 +48,6 @@ func (s *RMAStream) Calculate(x float64) float64 {
 	}
 	s.counter++
 
-	if s.counter < s.window {
-		// we can use x, but we need to use 0. to make the same behavior as the result from python pandas_ta
-		s.Slice.Push(0)
-	}
-
 	s.Slice.Push(tmp)
 	s.previous = tmp
 	return tmp

From 22a7232e8b5a657f75cbd01a2b1cef49cad7b9eb Mon Sep 17 00:00:00 2001
From: narumi <toucans-cutouts0f@icloud.com>
Date: Tue, 24 Oct 2023 16:37:44 +0800
Subject: [PATCH 11/42] fix duplicate rma value

---
 pkg/indicator/v2/rma.go | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/pkg/indicator/v2/rma.go b/pkg/indicator/v2/rma.go
index 4a6c8b327..b943cf6f8 100644
--- a/pkg/indicator/v2/rma.go
+++ b/pkg/indicator/v2/rma.go
@@ -34,23 +34,20 @@ func RMA2(source types.Float64Source, window int, adjust bool) *RMAStream {
 
 func (s *RMAStream) Calculate(x float64) float64 {
 	lambda := 1 / float64(s.window)
-	tmp := 0.0
 	if s.counter == 0 {
 		s.sum = 1
-		tmp = x
+		s.previous = x
 	} else {
 		if s.Adjust {
 			s.sum = s.sum*(1-lambda) + 1
-			tmp = s.previous + (x-s.previous)/s.sum
+			s.previous = s.previous + (x-s.previous)/s.sum
 		} else {
-			tmp = s.previous*(1-lambda) + x*lambda
+			s.previous = s.previous*(1-lambda) + x*lambda
 		}
 	}
 	s.counter++
 
-	s.Slice.Push(tmp)
-	s.previous = tmp
-	return tmp
+	return s.previous
 }
 
 func (s *RMAStream) Truncate() {

From 2a9fd10716b8c345b7606fdde49ec76c5eb6364a Mon Sep 17 00:00:00 2001
From: narumi <toucans-cutouts0f@icloud.com>
Date: Tue, 24 Oct 2023 16:38:05 +0800
Subject: [PATCH 12/42] add rma test cases

---
 pkg/indicator/rma_test.go    | 72 ++++++++++++++++++++++++++++++++++++
 pkg/indicator/v2/rma_test.go | 65 ++++++++++++++++++++++++++++++++
 2 files changed, 137 insertions(+)
 create mode 100644 pkg/indicator/rma_test.go
 create mode 100644 pkg/indicator/v2/rma_test.go

diff --git a/pkg/indicator/rma_test.go b/pkg/indicator/rma_test.go
new file mode 100644
index 000000000..6e14bcdb1
--- /dev/null
+++ b/pkg/indicator/rma_test.go
@@ -0,0 +1,72 @@
+package indicator
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+	"github.com/stretchr/testify/assert"
+)
+
+/*
+python
+
+import pandas as pd
+import pandas_ta as ta
+
+data = [40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84]
+
+close = pd.Series(data)
+result = ta.rma(close, length=14)
+print(result)
+*/
+func Test_RMA(t *testing.T) {
+	var bytes = []byte(`[40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84]`)
+	var values []fixedpoint.Value
+	err := json.Unmarshal(bytes, &values)
+	assert.NoError(t, err)
+
+	var kLines []types.KLine
+	for _, p := range values {
+		kLines = append(kLines, types.KLine{High: p, Low: p, Close: p})
+	}
+
+	tests := []struct {
+		name   string
+		window int
+		want   []float64
+	}{
+		{
+			name:   "test_binance_btcusdt_1h",
+			window: 14,
+			want: []float64{
+				40129.841000,
+				40041.830291,
+				39988.157743,
+				39958.803719,
+				39946.115094,
+				39958.296741,
+				39967.681562,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rma := RMA{
+				IntervalWindow: types.IntervalWindow{Window: tt.window},
+				Adjust:         true,
+			}
+			rma.CalculateAndUpdate(kLines)
+
+			if assert.Equal(t, len(tt.want), len(rma.Values)-tt.window+1) {
+				for i, v := range tt.want {
+					j := tt.window - 1 + i
+					got := rma.Values[j]
+					assert.InDelta(t, v, got, 0.01, "Expected rma.slice[%d] to be %v, but got %v", j, v, got)
+				}
+			}
+		})
+	}
+}
diff --git a/pkg/indicator/v2/rma_test.go b/pkg/indicator/v2/rma_test.go
new file mode 100644
index 000000000..b06605f75
--- /dev/null
+++ b/pkg/indicator/v2/rma_test.go
@@ -0,0 +1,65 @@
+package indicatorv2
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+/*
+python
+
+import pandas as pd
+import pandas_ta as ta
+
+data = [40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84]
+
+close = pd.Series(data)
+result = ta.rma(close, length=14)
+print(result)
+*/
+func Test_RMA2(t *testing.T) {
+	var bytes = []byte(`[40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84]`)
+	var values []float64
+	err := json.Unmarshal(bytes, &values)
+	assert.NoError(t, err)
+
+	prices := ClosePrices(nil)
+	for _, v := range values {
+		prices.Push(v)
+	}
+
+	tests := []struct {
+		name   string
+		window int
+		want   []float64
+	}{
+		{
+			name:   "test_binance_btcusdt_1h",
+			window: 14,
+			want: []float64{
+				40129.841000,
+				40041.830291,
+				39988.157743,
+				39958.803719,
+				39946.115094,
+				39958.296741,
+				39967.681562,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rma := RMA2(prices, tt.window, true)
+			if assert.Equal(t, len(tt.want), len(rma.Slice)-tt.window+1) {
+				for i, v := range tt.want {
+					j := tt.window - 1 + i
+					got := rma.Slice[j]
+					assert.InDelta(t, v, got, 0.01, "Expected rma.slice[%d] to be %v, but got %v", j, v, got)
+				}
+			}
+		})
+	}
+}

From 3e5869cab37f0dd45e1472ff258b3104dbfd5594 Mon Sep 17 00:00:00 2001
From: narumi <toucans-cutouts0f@icloud.com>
Date: Tue, 24 Oct 2023 17:03:40 +0800
Subject: [PATCH 13/42] remove zero padding from RMA

---
 pkg/indicator/rma.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/pkg/indicator/rma.go b/pkg/indicator/rma.go
index a06ad18af..69a7f4fbc 100644
--- a/pkg/indicator/rma.go
+++ b/pkg/indicator/rma.go
@@ -67,11 +67,6 @@ func (inc *RMA) Update(x float64) {
 	}
 	inc.counter++
 
-	if inc.counter < inc.Window {
-		inc.Values.Push(0)
-		return
-	}
-
 	inc.Values.Push(inc.tmp)
 	if len(inc.Values) > MaxNumOfRMA {
 		inc.Values = inc.Values[MaxNumOfRMATruncateSize-1:]

From 75575792e706f9139b02274cd961858a8da96bc5 Mon Sep 17 00:00:00 2001
From: Saksham Bhugra <85192629+sh4d0wy@users.noreply.github.com>
Date: Tue, 24 Oct 2023 22:40:47 +0530
Subject: [PATCH 14/42] Update CONTRIBUTING.md

---
 CONTRIBUTING.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c92962cdc..f25a27df0 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -31,19 +31,19 @@ Install pre-commit to check your changes before you commit:
 
 See <https://pre-commit.com/> for more details.
 
-For new large features, such as integrating binance futures contracts, please propose a discussion first before you start working on it.
+For new large features, such as integrating Binance futures contracts, please propose a discussion first before you start working on it.
 
 For new small features, you could open a pull request directly.
 
 For each contributor, you have chance to receive the BBG token through the polygon network.
 
-Each issue has its BBG label, by completing the issue with a pull request, you can get correspond amount of BBG.
+Each issue has its BBG label, by completing the issue with a pull request, you can get corresponding amount of BBG.
 
 ## Support
 
 ### By contributing pull requests
 
-Any pull request is welcome, documentation, format fixing, testing, features.
+Any pull request is welcome, documentation, format fixing, testing, and features.
 
 ### By registering account with referral ID
 
@@ -52,7 +52,7 @@ You may register your exchange account with my referral ID to support this proje
 - For MAX Exchange: <https://max.maicoin.com/signup?r=c7982718> (default commission rate to your account)
 - For Binance Exchange: <https://www.binancezh.com/en/register?ref=VGDGLT80> (5% commission back to your account)
 
-### By small amount cryptos
+### By small amount of cryptos
 
 - BTC address `3J6XQJNWT56amqz9Hz2BEVQ7W4aNmb5kiU`
 - USDT ERC20 address `0xeBcf7887A5b767DEb2e0C77E46A22c6Adc64E427`

From ab1bc998f9b5453d57870e79b56189b46b697532 Mon Sep 17 00:00:00 2001
From: chiahung <s199180626@gmail.com>
Date: Mon, 23 Oct 2023 15:55:55 +0800
Subject: [PATCH 15/42] FEATURE: prepare query trades funtion for new recover

---
 pkg/strategy/grid2/active_order_recover.go    |  16 --
 .../grid2/active_order_recover_test.go        |  74 ------
 pkg/strategy/grid2/recover.go                 | 143 +++++++++++
 pkg/strategy/grid2/recover_test.go            | 237 ++++++++++++++++++
 pkg/strategy/grid2/twin_order.go              |   5 +-
 pkg/strategy/grid2/twin_order_test.go         |   5 +
 6 files changed, 389 insertions(+), 91 deletions(-)
 create mode 100644 pkg/strategy/grid2/recover.go
 create mode 100644 pkg/strategy/grid2/recover_test.go

diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go
index 2ffdbb43c..5042084fc 100644
--- a/pkg/strategy/grid2/active_order_recover.go
+++ b/pkg/strategy/grid2/active_order_recover.go
@@ -2,7 +2,6 @@ package grid2
 
 import (
 	"context"
-	"strconv"
 	"time"
 
 	"github.com/c9s/bbgo/pkg/bbgo"
@@ -142,18 +141,3 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error {
 
 	return errs
 }
-
-func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64) error {
-	updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{
-		Symbol:  activeOrderBook.Symbol,
-		OrderID: strconv.FormatUint(orderID, 10),
-	})
-
-	if err != nil {
-		return err
-	}
-
-	activeOrderBook.Update(*updatedOrder)
-
-	return nil
-}
diff --git a/pkg/strategy/grid2/active_order_recover_test.go b/pkg/strategy/grid2/active_order_recover_test.go
index 5a72c05c0..dffdccc38 100644
--- a/pkg/strategy/grid2/active_order_recover_test.go
+++ b/pkg/strategy/grid2/active_order_recover_test.go
@@ -174,77 +174,3 @@ func TestSyncActiveOrders(t *testing.T) {
 		assert.Equal(types.OrderStatusNew, activeOrders[0].Status)
 	})
 }
-
-func TestSyncActiveOrder(t *testing.T) {
-	assert := assert.New(t)
-
-	mockCtrl := gomock.NewController(t)
-	defer mockCtrl.Finish()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	symbol := "ETHUSDT"
-
-	t.Run("sync filled order in active orderbook, active orderbook should remove this order", func(t *testing.T) {
-		mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl)
-		activeOrderbook := bbgo.NewActiveOrderBook(symbol)
-
-		order := types.Order{
-			OrderID: 1,
-			Status:  types.OrderStatusNew,
-			SubmitOrder: types.SubmitOrder{
-				Symbol: symbol,
-			},
-		}
-		activeOrderbook.Add(order)
-
-		updatedOrder := order
-		updatedOrder.Status = types.OrderStatusFilled
-
-		mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{
-			Symbol:  symbol,
-			OrderID: strconv.FormatUint(order.OrderID, 10),
-		}).Return(&updatedOrder, nil)
-
-		if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) {
-			return
-		}
-
-		// verify active orderbook
-		activeOrders := activeOrderbook.Orders()
-		assert.Equal(0, len(activeOrders))
-	})
-
-	t.Run("sync partial-filled order in active orderbook, active orderbook should still keep this order", func(t *testing.T) {
-		mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl)
-		activeOrderbook := bbgo.NewActiveOrderBook(symbol)
-
-		order := types.Order{
-			OrderID: 1,
-			Status:  types.OrderStatusNew,
-			SubmitOrder: types.SubmitOrder{
-				Symbol: symbol,
-			},
-		}
-		activeOrderbook.Add(order)
-
-		updatedOrder := order
-		updatedOrder.Status = types.OrderStatusPartiallyFilled
-
-		mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{
-			Symbol:  symbol,
-			OrderID: strconv.FormatUint(order.OrderID, 10),
-		}).Return(&updatedOrder, nil)
-
-		if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) {
-			return
-		}
-
-		// verify active orderbook
-		activeOrders := activeOrderbook.Orders()
-		assert.Equal(1, len(activeOrders))
-		assert.Equal(order.OrderID, activeOrders[0].OrderID)
-		assert.Equal(updatedOrder.Status, activeOrders[0].Status)
-	})
-}
diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go
new file mode 100644
index 000000000..623966e60
--- /dev/null
+++ b/pkg/strategy/grid2/recover.go
@@ -0,0 +1,143 @@
+package grid2
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/c9s/bbgo/pkg/bbgo"
+	"github.com/c9s/bbgo/pkg/exchange/retry"
+	"github.com/c9s/bbgo/pkg/types"
+	"github.com/pkg/errors"
+)
+
+/*
+  	Background knowledge
+  	1. active orderbook add orders only when receive new order event or call Add/Update method manually
+  	2. active orderbook remove orders only when receive filled/cancelled event or call Remove/Update method manually
+  	As a result
+  	1. at the same twin-order-price, there is order in open orders but not in active orderbook
+  		- not receive new order event
+		  	=> add order into active orderbook
+  	2. at the same twin-order-price, there is order in active orderbook but not in open orders
+  		- not receive filled event
+			=> query the filled order and call Update method
+  	3. at the same twin-order-price, there is no order in open orders and no order in active orderbook
+		- failed to create the order
+			=> query the last order from trades to emit filled, and it will submit again
+		- not receive new order event and the order filled before we find it.
+			=> query the untracked order (also is the last order) from trades to emit filled and it will submit the reversed order
+  	4. at the same twin-order-price, there are different orders in open orders and active orderbook
+	  	- should not happen !!!
+		  	=> log error
+	5. at the same twin-order-price, there is the same order in open orders and active orderbook
+		- normal case
+			=> no need to do anything
+	After killing pod, active orderbook must be empty. we can think it is the same as not receive new event.
+	Process
+	1. build twin orderbook with pins and open orders.
+	2. build twin orderbook with pins and active orders.
+	3. compare above twin orderbooks to add open orders into active orderbook and update active orders.
+	4. run grid recover to make sure all the twin price has its order.
+*/
+
+func buildTwinOrderBook(pins []Pin, orders []types.Order) (*TwinOrderBook, error) {
+	book := newTwinOrderBook(pins)
+
+	for _, order := range orders {
+		if err := book.AddOrder(order); err != nil {
+			return nil, err
+		}
+	}
+
+	return book, nil
+}
+
+func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64) error {
+	updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{
+		Symbol:  activeOrderBook.Symbol,
+		OrderID: strconv.FormatUint(orderID, 10),
+	})
+
+	if err != nil {
+		return err
+	}
+
+	activeOrderBook.Update(*updatedOrder)
+
+	return nil
+}
+
+func queryTradesToUpdateTwinOrderBook(
+	ctx context.Context,
+	symbol string,
+	twinOrderBook *TwinOrderBook,
+	queryTradesService types.ExchangeTradeHistoryService,
+	queryOrderService types.ExchangeOrderQueryService,
+	existedOrders *types.SyncOrderMap,
+	since, until time.Time,
+	logger func(format string, args ...interface{})) error {
+	if twinOrderBook == nil {
+		return fmt.Errorf("twin orderbook should not be nil, please check it")
+	}
+
+	var fromTradeID uint64 = 0
+	var limit int64 = 1000
+	for {
+		trades, err := queryTradesService.QueryTrades(ctx, symbol, &types.TradeQueryOptions{
+			StartTime:   &since,
+			EndTime:     &until,
+			LastTradeID: fromTradeID,
+			Limit:       limit,
+		})
+
+		if err != nil {
+			return errors.Wrapf(err, "failed to query trades to recover the grid")
+		}
+
+		if logger != nil {
+			logger("QueryTrades from %s <-> %s (from: %d) return %d trades", since, until, fromTradeID, len(trades))
+		}
+
+		for _, trade := range trades {
+			if trade.Time.After(until) {
+				return nil
+			}
+
+			if logger != nil {
+				logger(trade.String())
+			}
+
+			if existedOrders.Exists(trade.OrderID) {
+				// already queries, skip
+				continue
+			}
+			order, err := retry.QueryOrderUntilSuccessful(ctx, queryOrderService, types.OrderQuery{
+				Symbol:  trade.Symbol,
+				OrderID: strconv.FormatUint(trade.OrderID, 10),
+			})
+
+			if err != nil {
+				return errors.Wrapf(err, "failed to query order by trade (trade id: %d, order id: %d)", trade.ID, trade.OrderID)
+			}
+
+			if logger != nil {
+				logger(order.String())
+			}
+			// avoid query this order again
+			existedOrders.Add(*order)
+			// add 1 to avoid duplicate
+			fromTradeID = trade.ID + 1
+
+			if err := twinOrderBook.AddOrder(*order); err != nil {
+				return errors.Wrapf(err, "failed to add queried order into twin orderbook")
+			}
+		}
+
+		// stop condition
+		if int64(len(trades)) < limit {
+			return nil
+		}
+	}
+}
diff --git a/pkg/strategy/grid2/recover_test.go b/pkg/strategy/grid2/recover_test.go
new file mode 100644
index 000000000..bdfd191ee
--- /dev/null
+++ b/pkg/strategy/grid2/recover_test.go
@@ -0,0 +1,237 @@
+package grid2
+
+import (
+	"context"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/c9s/bbgo/pkg/bbgo"
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+	"github.com/c9s/bbgo/pkg/types/mocks"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+)
+
+func newStrategy(t *TestData) *Strategy {
+	s := t.Strategy
+	s.Debug = true
+	s.Initialize()
+	s.Market = t.Market
+	s.Position = types.NewPositionFromMarket(t.Market)
+	s.orderExecutor = bbgo.NewGeneralOrderExecutor(&bbgo.ExchangeSession{}, t.Market.Symbol, ID, s.InstanceID(), s.Position)
+	return &s
+}
+
+func TestBuildTwinOrderBook(t *testing.T) {
+	assert := assert.New(t)
+
+	pins := []Pin{
+		Pin(fixedpoint.NewFromInt(200)),
+		Pin(fixedpoint.NewFromInt(300)),
+		Pin(fixedpoint.NewFromInt(500)),
+		Pin(fixedpoint.NewFromInt(400)),
+		Pin(fixedpoint.NewFromInt(100)),
+	}
+	t.Run("build twin orderbook with no order", func(t *testing.T) {
+		b, err := buildTwinOrderBook(pins, nil)
+		if !assert.NoError(err) {
+			return
+		}
+
+		assert.Equal(0, b.Size())
+		assert.Nil(b.GetTwinOrder(fixedpoint.NewFromInt(100)))
+		assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(200)).Exist())
+		assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(300)).Exist())
+		assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(400)).Exist())
+		assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(500)).Exist())
+	})
+
+	t.Run("build twin orderbook with some valid orders", func(t *testing.T) {
+		orders := []types.Order{
+			{
+				OrderID: 1,
+				SubmitOrder: types.SubmitOrder{
+					Side:  types.SideTypeBuy,
+					Price: fixedpoint.NewFromInt(100),
+				},
+			},
+			{
+				OrderID: 5,
+				SubmitOrder: types.SubmitOrder{
+					Side:  types.SideTypeSell,
+					Price: fixedpoint.NewFromInt(500),
+				},
+			},
+		}
+		b, err := buildTwinOrderBook(pins, orders)
+		if !assert.NoError(err) {
+			return
+		}
+
+		assert.Equal(2, b.Size())
+		assert.Equal(2, b.EmptyTwinOrderSize())
+		assert.Nil(b.GetTwinOrder(fixedpoint.NewFromInt(100)))
+		assert.True(b.GetTwinOrder(fixedpoint.NewFromInt(200)).Exist())
+		assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(300)).Exist())
+		assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(400)).Exist())
+		assert.True(b.GetTwinOrder(fixedpoint.NewFromInt(500)).Exist())
+	})
+
+	t.Run("build twin orderbook with invalid orders", func(t *testing.T) {})
+}
+
+func TestSyncActiveOrder(t *testing.T) {
+	assert := assert.New(t)
+
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	symbol := "ETHUSDT"
+
+	t.Run("sync filled order in active orderbook, active orderbook should remove this order", func(t *testing.T) {
+		mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl)
+		activeOrderbook := bbgo.NewActiveOrderBook(symbol)
+
+		order := types.Order{
+			OrderID: 1,
+			Status:  types.OrderStatusNew,
+			SubmitOrder: types.SubmitOrder{
+				Symbol: symbol,
+			},
+		}
+		activeOrderbook.Add(order)
+
+		updatedOrder := order
+		updatedOrder.Status = types.OrderStatusFilled
+
+		mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{
+			Symbol:  symbol,
+			OrderID: strconv.FormatUint(order.OrderID, 10),
+		}).Return(&updatedOrder, nil)
+
+		if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) {
+			return
+		}
+
+		// verify active orderbook
+		activeOrders := activeOrderbook.Orders()
+		assert.Equal(0, len(activeOrders))
+	})
+
+	t.Run("sync partial-filled order in active orderbook, active orderbook should still keep this order", func(t *testing.T) {
+		mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl)
+		activeOrderbook := bbgo.NewActiveOrderBook(symbol)
+
+		order := types.Order{
+			OrderID: 1,
+			Status:  types.OrderStatusNew,
+			SubmitOrder: types.SubmitOrder{
+				Symbol: symbol,
+			},
+		}
+		activeOrderbook.Add(order)
+
+		updatedOrder := order
+		updatedOrder.Status = types.OrderStatusPartiallyFilled
+
+		mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{
+			Symbol:  symbol,
+			OrderID: strconv.FormatUint(order.OrderID, 10),
+		}).Return(&updatedOrder, nil)
+
+		if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) {
+			return
+		}
+
+		// verify active orderbook
+		activeOrders := activeOrderbook.Orders()
+		assert.Equal(1, len(activeOrders))
+		assert.Equal(order.OrderID, activeOrders[0].OrderID)
+		assert.Equal(updatedOrder.Status, activeOrders[0].Status)
+	})
+}
+
+func TestQueryTradesToUpdateTwinOrderBook(t *testing.T) {
+	assert := assert.New(t)
+
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	symbol := "ETHUSDT"
+	pins := []Pin{
+		Pin(fixedpoint.NewFromInt(100)),
+		Pin(fixedpoint.NewFromInt(200)),
+		Pin(fixedpoint.NewFromInt(300)),
+		Pin(fixedpoint.NewFromInt(400)),
+		Pin(fixedpoint.NewFromInt(500)),
+	}
+
+	t.Run("query trades and update twin orderbook successfully in one page", func(t *testing.T) {
+		book := newTwinOrderBook(pins)
+		mockTradeHistoryService := mocks.NewMockExchangeTradeHistoryService(mockCtrl)
+		mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl)
+
+		trades := []types.Trade{
+			{
+				ID:      1,
+				OrderID: 1,
+				Symbol:  symbol,
+				Time:    types.Time(time.Now().Add(-2 * time.Hour)),
+			},
+			{
+				ID:      2,
+				OrderID: 2,
+				Symbol:  symbol,
+				Time:    types.Time(time.Now().Add(-1 * time.Hour)),
+			},
+		}
+		orders := []types.Order{
+			{
+				OrderID: 1,
+				Status:  types.OrderStatusNew,
+				SubmitOrder: types.SubmitOrder{
+					Symbol: symbol,
+					Side:   types.SideTypeBuy,
+					Price:  fixedpoint.NewFromInt(100),
+				},
+			},
+			{
+				OrderID: 2,
+				Status:  types.OrderStatusFilled,
+				SubmitOrder: types.SubmitOrder{
+					Symbol: symbol,
+					Side:   types.SideTypeSell,
+					Price:  fixedpoint.NewFromInt(500),
+				},
+			},
+		}
+		mockTradeHistoryService.EXPECT().QueryTrades(gomock.Any(), gomock.Any(), gomock.Any()).Return(trades, nil).Times(1)
+		mockOrderQueryService.EXPECT().QueryOrder(gomock.Any(), types.OrderQuery{
+			Symbol:  symbol,
+			OrderID: "1",
+		}).Return(&orders[0], nil)
+		mockOrderQueryService.EXPECT().QueryOrder(gomock.Any(), types.OrderQuery{
+			Symbol:  symbol,
+			OrderID: "2",
+		}).Return(&orders[1], nil)
+
+		assert.Equal(0, book.Size())
+		if !assert.NoError(queryTradesToUpdateTwinOrderBook(ctx, symbol, book, mockTradeHistoryService, mockOrderQueryService, book.SyncOrderMap(), time.Now().Add(-24*time.Hour), time.Now(), nil)) {
+			return
+		}
+
+		assert.Equal(2, book.Size())
+		assert.True(book.GetTwinOrder(fixedpoint.NewFromInt(200)).Exist())
+		assert.Equal(orders[0].OrderID, book.GetTwinOrder(fixedpoint.NewFromInt(200)).GetOrder().OrderID)
+		assert.True(book.GetTwinOrder(fixedpoint.NewFromInt(500)).Exist())
+		assert.Equal(orders[1].OrderID, book.GetTwinOrder(fixedpoint.NewFromInt(500)).GetOrder().OrderID)
+	})
+}
diff --git a/pkg/strategy/grid2/twin_order.go b/pkg/strategy/grid2/twin_order.go
index ccc9bfa4f..f3a093b95 100644
--- a/pkg/strategy/grid2/twin_order.go
+++ b/pkg/strategy/grid2/twin_order.go
@@ -154,7 +154,10 @@ func newTwinOrderBook(pins []Pin) *TwinOrderBook {
 	pinIdx := make(map[fixedpoint.Value]int)
 	m := make(map[fixedpoint.Value]*TwinOrder)
 	for i, pin := range v {
-		m[pin] = &TwinOrder{}
+		// we use sell price for twin orderbook's price, so we skip the first pin as price
+		if i > 0 {
+			m[pin] = &TwinOrder{}
+		}
 		pinIdx[pin] = i
 	}
 
diff --git a/pkg/strategy/grid2/twin_order_test.go b/pkg/strategy/grid2/twin_order_test.go
index d6204ee94..47395303f 100644
--- a/pkg/strategy/grid2/twin_order_test.go
+++ b/pkg/strategy/grid2/twin_order_test.go
@@ -23,6 +23,11 @@ func TestTwinOrderBook(t *testing.T) {
 	assert.Equal(4, book.EmptyTwinOrderSize())
 	for _, pin := range pins {
 		twinOrder := book.GetTwinOrder(fixedpoint.Value(pin))
+		if fixedpoint.NewFromInt(1) == fixedpoint.Value(pin) {
+			assert.Nil(twinOrder)
+			continue
+		}
+
 		if !assert.NotNil(twinOrder) {
 			continue
 		}

From 7e532f55751e975cb47791dc8509e3064cb601f2 Mon Sep 17 00:00:00 2001
From: Rohan Kumar <130988597+rohan37kumar@users.noreply.github.com>
Date: Wed, 25 Oct 2023 12:49:33 +0530
Subject: [PATCH 16/42] Updated README.md

---
 README.md | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 85ab0f27c..12af0d06f 100644
--- a/README.md
+++ b/README.md
@@ -30,14 +30,14 @@ You can use BBGO's trading unit and back-test unit to implement your own strateg
 
 ### Trading Unit Developers 🧑‍💻
 
-You can use BBGO's underlying common exchange API, currently it supports 4+ major exchanges, so you don't have to repeat
+You can use BBGO's underlying common exchange API, currently, it supports 4+ major exchanges, so you don't have to repeat
 the implementation.
 
 ## Features
 
 - Exchange abstraction interface.
-- Stream integration (user data websocket, market data websocket).
-- Real-time orderBook integration through websocket.
+- Stream integration (user data web socket, market data web socket).
+- Real-time orderBook integration through a web socket.
 - TWAP order execution support. See [TWAP Order Execution](./doc/topics/twap.md)
 - PnL calculation.
 - Slack/Telegram notification.
@@ -177,7 +177,7 @@ bash <(curl -s https://raw.githubusercontent.com/c9s/bbgo/main/scripts/download.
 
 Or refer to the [Release Page](https://github.com/c9s/bbgo/releases) and download manually.
 
-Since v2, we've added new float point implementation from dnum to support decimals with higher precision. To download &
+Since v2, we've added a new float point implementation from dnum to support decimals with higher precision. To download &
 setup, please refer to [Dnum Installation](doc/topics/dnum-binary.md)
 
 ### One-click Linode StackScript
@@ -319,7 +319,7 @@ You can only use one database driver MySQL or SQLite to store your trading data.
 
 #### Configure MySQL Database
 
-To use MySQL database for data syncing, first you need to install your mysql server:
+To use MySQL database for data syncing, first, you need to install your mysql server:
 
 ```sh
 # For Ubuntu Linux
@@ -427,7 +427,7 @@ See [Developing Strategy](./doc/topics/developing-strategy.md)
 
 ## Write your own private strategy
 
-Create your go package, and initialize the repository with `go mod` and add bbgo as a dependency:
+Create your go package, initialize the repository with `go mod`, and add bbgo as a dependency:
 
 ```sh
 go mod init
@@ -550,7 +550,7 @@ following types could be injected automatically:
 2. Allocate and initialize exchange sessions.
 3. Add exchange sessions to the environment (the data layer).
 4. Use the given environment to initialize the trader object (the logic layer).
-5. The trader initializes the environment and start the exchange connections.
+5. The trader initializes the environment and starts the exchange connections.
 6. Call strategy.Run() method sequentially.
 
 ## Exchange API Examples
@@ -567,7 +567,7 @@ maxRest := maxapi.NewRestClient(maxapi.ProductionAPIURL)
 maxRest.Auth(key, secret)
 ```
 
-Creating user data stream to get the orderbook (depth):
+Creating user data stream to get the order book (depth):
 
 ```go
 stream := max.NewStream(key, secret)
@@ -591,7 +591,7 @@ streambook.BindStream(stream)
 
 1. Click the "Fork" button from the GitHub repository.
 2. Clone your forked repository into `$GOPATH/github.com/c9s/bbgo`.
-3. Change directory into `$GOPATH/github.com/c9s/bbgo`.
+3. Change the directory into `$GOPATH/github.com/c9s/bbgo`.
 4. Create a branch and start your development.
 5. Test your changes.
 6. Push your changes to your fork.
@@ -616,13 +616,13 @@ make embed && go run -tags web ./cmd/bbgo-lorca
 ### What's Position?
 
 - Base Currency & Quote Currency <https://www.ig.com/au/glossary-trading-terms/base-currency-definition>
-- How to calculate average cost? <https://www.janushenderson.com/en-us/investor/planning/calculate-average-cost/>
+- How to calculate the average cost? <https://www.janushenderson.com/en-us/investor/planning/calculate-average-cost/>
 
 ### Looking For A New Strategy?
 
-You can write an article about BBGO in any topic, in 750-1500 words for exchange, and I can implement the strategy for
-you (depends on the complexity and efforts). If you're interested in, DM me in telegram <https://t.me/c123456789s> or
-twitter <https://twitter.com/c9s>, we can discuss.
+You can write an article about BBGO on any topic, in 750-1500 words for exchange, and I can implement the strategy for
+you (depending on the complexity and efforts). If you're interested in, DM me in telegram <https://t.me/c123456789s> or
+twitter <https://twitter.com/c9s>, and we can discuss.
 
 ### Adding New Crypto Exchange support?
 

From c611cfe73baa7025468ead2da47d8b650c26b601 Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Wed, 25 Oct 2023 21:30:54 +0800
Subject: [PATCH 17/42]  pkg/exchange: add a jumpIfEmpty to batch trade option

---
 pkg/exchange/batch/option.go | 12 ++++++++++++
 pkg/exchange/batch/trade.go  |  6 +++++-
 2 files changed, 17 insertions(+), 1 deletion(-)
 create mode 100644 pkg/exchange/batch/option.go

diff --git a/pkg/exchange/batch/option.go b/pkg/exchange/batch/option.go
new file mode 100644
index 000000000..67f18e608
--- /dev/null
+++ b/pkg/exchange/batch/option.go
@@ -0,0 +1,12 @@
+package batch
+
+import "time"
+
+type Option func(query *AsyncTimeRangedBatchQuery)
+
+// JumpIfEmpty jump the startTime + duration when the result is empty
+func JumpIfEmpty(duration time.Duration) Option {
+	return func(query *AsyncTimeRangedBatchQuery) {
+		query.JumpIfEmpty = duration
+	}
+}
diff --git a/pkg/exchange/batch/trade.go b/pkg/exchange/batch/trade.go
index 1c91da777..4fce26b65 100644
--- a/pkg/exchange/batch/trade.go
+++ b/pkg/exchange/batch/trade.go
@@ -17,7 +17,7 @@ type TradeBatchQuery struct {
 	types.ExchangeTradeHistoryService
 }
 
-func (e TradeBatchQuery) Query(ctx context.Context, symbol string, options *types.TradeQueryOptions) (c chan types.Trade, errC chan error) {
+func (e TradeBatchQuery) Query(ctx context.Context, symbol string, options *types.TradeQueryOptions, opts ...Option) (c chan types.Trade, errC chan error) {
 	if options.EndTime == nil {
 		now := time.Now()
 		options.EndTime = &now
@@ -45,6 +45,10 @@ func (e TradeBatchQuery) Query(ctx context.Context, symbol string, options *type
 		JumpIfEmpty: 24 * time.Hour,
 	}
 
+	for _, opt := range opts {
+		opt(query)
+	}
+
 	c = make(chan types.Trade, 100)
 	errC = query.Query(ctx, c, startTime, endTime)
 	return c, errC

From 881db49b70306b1b2f7be75d032831c60ae4461b Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Wed, 25 Oct 2023 21:36:26 +0800
Subject: [PATCH 18/42] pkg/exchange: rename tradeRateLimiter to 
 queryOrderTradeRateLimiter

---
 pkg/exchange/bybit/exchange.go | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/pkg/exchange/bybit/exchange.go b/pkg/exchange/bybit/exchange.go
index e1c3545c5..7cec992c6 100644
--- a/pkg/exchange/bybit/exchange.go
+++ b/pkg/exchange/bybit/exchange.go
@@ -26,15 +26,15 @@ const (
 )
 
 // https://bybit-exchange.github.io/docs/zh-TW/v5/rate-limit
-// sharedRateLimiter indicates that the API belongs to the public API.
-//
-// The default order limiter apply 5 requests per second and a 5 initial bucket
-// this includes QueryMarkets, QueryTicker, QueryAccountBalances, GetFeeRates
+// GET/POST method (shared): 120 requests per second for 5 consecutive seconds
 var (
-	sharedRateLimiter       = rate.NewLimiter(rate.Every(time.Second/5), 5)
-	tradeRateLimiter        = rate.NewLimiter(rate.Every(time.Second/5), 5)
-	orderRateLimiter        = rate.NewLimiter(rate.Every(100*time.Millisecond), 10)
-	closedOrderQueryLimiter = rate.NewLimiter(rate.Every(time.Second), 1)
+	// sharedRateLimiter indicates that the API belongs to the public API.
+	// The default order limiter apply 5 requests per second and a 5 initial bucket
+	// this includes QueryMarkets, QueryTicker, QueryAccountBalances, GetFeeRates
+	sharedRateLimiter          = rate.NewLimiter(rate.Every(time.Second/5), 5)
+	queryOrderTradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5)
+	orderRateLimiter           = rate.NewLimiter(rate.Every(time.Second/10), 10)
+	closedOrderQueryLimiter    = rate.NewLimiter(rate.Every(time.Second), 1)
 
 	log = logrus.WithFields(logrus.Fields{
 		"exchange": "bybit",
@@ -159,7 +159,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [
 			req = req.Cursor(cursor)
 		}
 
-		if err = tradeRateLimiter.Wait(ctx); err != nil {
+		if err = queryOrderTradeRateLimiter.Wait(ctx); err != nil {
 			return nil, fmt.Errorf("place order rate limiter wait error: %w", err)
 		}
 		res, err := req.Do(ctx)
@@ -232,7 +232,7 @@ func (e *Exchange) QueryOrderTrades(ctx context.Context, q types.OrderQuery) (tr
 		req.Symbol(q.Symbol)
 	}
 
-	if err := tradeRateLimiter.Wait(ctx); err != nil {
+	if err := queryOrderTradeRateLimiter.Wait(ctx); err != nil {
 		return nil, fmt.Errorf("trade rate limiter wait error: %w", err)
 	}
 	response, err := req.Do(ctx)
@@ -463,7 +463,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type
 	}
 	req.Limit(limit)
 
-	if err := tradeRateLimiter.Wait(ctx); err != nil {
+	if err := queryOrderTradeRateLimiter.Wait(ctx); err != nil {
 		return nil, fmt.Errorf("trade rate limiter wait error: %w", err)
 	}
 	response, err := req.Do(ctx)

From 55d444d86a85046693403d2d4276ccc3c742e7e7 Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Thu, 26 Oct 2023 09:31:25 +0800
Subject: [PATCH 19/42] pkg/exchange: add jumpIfEmpty opts to closed order
 batch query

---
 pkg/exchange/batch/closedorders.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pkg/exchange/batch/closedorders.go b/pkg/exchange/batch/closedorders.go
index 51d12f5ff..77e37690b 100644
--- a/pkg/exchange/batch/closedorders.go
+++ b/pkg/exchange/batch/closedorders.go
@@ -12,7 +12,7 @@ type ClosedOrderBatchQuery struct {
 	types.ExchangeTradeHistoryService
 }
 
-func (q *ClosedOrderBatchQuery) Query(ctx context.Context, symbol string, startTime, endTime time.Time, lastOrderID uint64) (c chan types.Order, errC chan error) {
+func (q *ClosedOrderBatchQuery) Query(ctx context.Context, symbol string, startTime, endTime time.Time, lastOrderID uint64, opts ...Option) (c chan types.Order, errC chan error) {
 	query := &AsyncTimeRangedBatchQuery{
 		Type: types.Order{},
 		Q: func(startTime, endTime time.Time) (interface{}, error) {
@@ -32,6 +32,10 @@ func (q *ClosedOrderBatchQuery) Query(ctx context.Context, symbol string, startT
 		JumpIfEmpty: 30 * 24 * time.Hour,
 	}
 
+	for _, opt := range opts {
+		opt(query)
+	}
+
 	c = make(chan types.Order, 100)
 	errC = query.Query(ctx, c, startTime, endTime)
 	return c, errC

From 7c27cb9801df36c2b7f5d925369a8b020e1bdf9a Mon Sep 17 00:00:00 2001
From: Surav Shrestha <suravshresth@gmail.com>
Date: Thu, 26 Oct 2023 10:24:54 +0545
Subject: [PATCH 20/42] docs: fix typos in
 doc/development/adding-new-exchange.md

---
 doc/development/adding-new-exchange.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/development/adding-new-exchange.md b/doc/development/adding-new-exchange.md
index 6f0ae8075..1eb45a2ba 100644
--- a/doc/development/adding-new-exchange.md
+++ b/doc/development/adding-new-exchange.md
@@ -58,7 +58,7 @@ Stream
     - [ ] Public trade message parser (optional)
     - [ ] Ticker message parser (optional)
 - [ ] ping/pong handling. (you can reuse the existing types.StandardStream)
-- [ ] heart-beat hanlding or keep-alive handling. (already included in types.StandardStream)
+- [ ] heart-beat handling or keep-alive handling. (already included in types.StandardStream)
 - [ ] handling reconnect. (already included in types.StandardStream)
 
 Database

From 41896646270e7c699157cc52556291bb9e8b5f3a Mon Sep 17 00:00:00 2001
From: Surav Shrestha <suravshresth@gmail.com>
Date: Thu, 26 Oct 2023 10:25:09 +0545
Subject: [PATCH 21/42] docs: fix typos in doc/development/release-process.md

---
 doc/development/release-process.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/development/release-process.md b/doc/development/release-process.md
index 89efb1b35..eaf33d8f8 100644
--- a/doc/development/release-process.md
+++ b/doc/development/release-process.md
@@ -40,7 +40,7 @@ Run the following command to create the release:
 make version VERSION=v1.20.2
 ```
 
-The above command wilL:
+The above command will:
 
 - Update and compile the migration scripts into go files.
 - Bump the version name in the go code.

From fc9ce53747133fc03d3881538148b73fd3b78fe2 Mon Sep 17 00:00:00 2001
From: Surav Shrestha <suravshresth@gmail.com>
Date: Thu, 26 Oct 2023 10:25:22 +0545
Subject: [PATCH 22/42] docs: fix typos in doc/development/series.md

---
 doc/development/series.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/development/series.md b/doc/development/series.md
index 0f23756fc..ec9597377 100644
--- a/doc/development/series.md
+++ b/doc/development/series.md
@@ -25,7 +25,7 @@ type BoolSeries interface {
 }
 ```
 
-Series were used almost everywhere in indicators to return the calculated numeric results, but the use of BoolSeries is quite limited. At this moment, we only use BoolSeries to check if some condition is fullfilled at some timepoint. For example, in `CrossOver` and `CrossUnder` functions if `Last()` returns true, then there might be a cross event happend on the curves at the moment.
+Series were used almost everywhere in indicators to return the calculated numeric results, but the use of BoolSeries is quite limited. At this moment, we only use BoolSeries to check if some condition is fulfilled at some timepoint. For example, in `CrossOver` and `CrossUnder` functions if `Last()` returns true, then there might be a cross event happened on the curves at the moment.
 
 #### Expected Implementation
 
@@ -44,7 +44,7 @@ and if any of the method in the interface not been implemented, this would gener
 
 #### Extended Series
 
-Instead of simple Series interface, we have `types.SeriesExtend` interface that enriches the functionality of `types.Series`. An indicator struct could simply be extended to `types.SeriesExtend` type by embedding anonymous struct `types.SeriesBase`, and instanced by `types.NewSeries()` function. The `types.SeriesExtend` interface binds commonly used functions, such as `Add`, `Reverse`, `Shfit`, `Covariance` and `Entropy`, to the original `types.Series` object. Please check [pkg/types/seriesbase_imp.go](../../pkg/types/seriesbase_imp.go) for the extendable functions.
+Instead of simple Series interface, we have `types.SeriesExtend` interface that enriches the functionality of `types.Series`. An indicator struct could simply be extended to `types.SeriesExtend` type by embedding anonymous struct `types.SeriesBase`, and instanced by `types.NewSeries()` function. The `types.SeriesExtend` interface binds commonly used functions, such as `Add`, `Reverse`, `Shift`, `Covariance` and `Entropy`, to the original `types.Series` object. Please check [pkg/types/seriesbase_imp.go](../../pkg/types/seriesbase_imp.go) for the extendable functions.
 
 Example:
 

From f31d829294595f61263e1b65b3b7afb3fe66a5ec Mon Sep 17 00:00:00 2001
From: chiahung <s199180626@gmail.com>
Date: Tue, 24 Oct 2023 13:02:34 +0800
Subject: [PATCH 23/42] FEAUTRE: merge grid recover and active orders recover

---
 pkg/strategy/grid2/recover.go | 196 +++++++++++++++++++++++++++++++++-
 1 file changed, 195 insertions(+), 1 deletion(-)

diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go
index 623966e60..e9be4866d 100644
--- a/pkg/strategy/grid2/recover.go
+++ b/pkg/strategy/grid2/recover.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/c9s/bbgo/pkg/bbgo"
 	"github.com/c9s/bbgo/pkg/exchange/retry"
+	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/types"
 	"github.com/pkg/errors"
 )
@@ -42,6 +43,198 @@ import (
 	4. run grid recover to make sure all the twin price has its order.
 */
 
+func (s *Strategy) recover(ctx context.Context) error {
+	historyService, implemented := s.session.Exchange.(types.ExchangeTradeHistoryService)
+	// if the exchange doesn't support ExchangeTradeHistoryService, do not run recover
+	if !implemented {
+		s.logger.Warn("ExchangeTradeHistoryService is not implemented, can not recover grid")
+		return nil
+	}
+
+	activeOrderBook := s.orderExecutor.ActiveMakerOrders()
+	activeOrders := activeOrderBook.Orders()
+
+	openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol)
+	if err != nil {
+		return err
+	}
+
+	// check if it's new strategy or need to recover
+	if len(activeOrders) == 0 && len(openOrders) == 0 && s.GridProfitStats.InitialOrderID == 0 {
+		// even though there is no open orders and initial orderID is 0
+		// we still need to query trades to make sure if we need to recover or not
+		trades, err := historyService.QueryTrades(ctx, s.Symbol, &types.TradeQueryOptions{
+			// from 1, because some API will ignore 0 last trade id
+			LastTradeID: 1,
+			// if there is any trades, we need to recover.
+			Limit: 1,
+		})
+
+		if err != nil {
+			return errors.Wrapf(err, "unable to query trades when recovering")
+		}
+
+		if len(trades) == 0 {
+			s.logger.Info("no open order, no active order, no trade, it's a new strategy so no need to recover")
+			return nil
+		}
+	}
+
+	s.logger.Info("start recovering")
+
+	if s.getGrid() == nil {
+		s.setGrid(s.newGrid())
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	pins := s.getGrid().Pins
+
+	activeOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, activeOrders)
+	openOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, openOrders)
+
+	s.logger.Infof("active orders' twin orderbook\n%s", activeOrdersInTwinOrderBook.String())
+	s.logger.Infof("open orders in twin orderbook\n%s", openOrdersInTwinOrderBook.String())
+
+	// remove index 0, because twin orderbook's price is from the second one
+	pins = pins[1:]
+	var noTwinOrderPins []fixedpoint.Value
+
+	for _, pin := range pins {
+		v := fixedpoint.Value(pin)
+		activeOrder := activeOrdersInTwinOrderBook.GetTwinOrder(v)
+		openOrder := openOrdersInTwinOrderBook.GetTwinOrder(v)
+		if activeOrder == nil || openOrder == nil {
+			return fmt.Errorf("there is no any twin order at this pin, can not recover")
+		}
+
+		var activeOrderID uint64 = 0
+		if activeOrder.Exist() {
+			activeOrderID = activeOrder.GetOrder().OrderID
+		}
+
+		var openOrderID uint64 = 0
+		if openOrder.Exist() {
+			openOrderID = openOrder.GetOrder().OrderID
+		}
+
+		// case 3
+		if activeOrderID == 0 && openOrderID == 0 {
+			noTwinOrderPins = append(noTwinOrderPins, v)
+			continue
+		}
+
+		// case 1
+		if activeOrderID == 0 {
+			activeOrderBook.Add(openOrder.GetOrder())
+			// also add open orders into active order's twin orderbook, we will use this active orderbook to recover empty price grid
+			activeOrdersInTwinOrderBook.AddTwinOrder(v, openOrder)
+			continue
+		}
+
+		// case 2
+		if openOrderID == 0 {
+			syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, activeOrder.GetOrder().OrderID)
+			continue
+		}
+
+		// case 4
+		if activeOrderID != openOrderID {
+			return fmt.Errorf("there are two different orders in the same pin, can not recover")
+		}
+
+		// case 5
+		// do nothing
+	}
+
+	s.logger.Infof("twin orderbook after adding open orders\n%s", activeOrdersInTwinOrderBook.String())
+
+	if err := s.recoverEmptyGridOnTwinOrderBook(ctx, activeOrdersInTwinOrderBook, historyService, s.orderQueryService); err != nil {
+		s.logger.WithError(err).Error("failed to recover empty grid")
+		return err
+	}
+
+	s.logger.Infof("twin orderbook after recovering\n%s", activeOrdersInTwinOrderBook.String())
+
+	if activeOrdersInTwinOrderBook.EmptyTwinOrderSize() > 0 {
+		return fmt.Errorf("there is still empty grid in twin orderbook")
+	}
+
+	for _, pin := range noTwinOrderPins {
+		twinOrder := activeOrdersInTwinOrderBook.GetTwinOrder(pin)
+		if twinOrder == nil {
+			return fmt.Errorf("should not get nil twin order after recovering empty grid, check it")
+		}
+
+		if !twinOrder.Exist() {
+			return fmt.Errorf("should not get empty twin order after recovering empty grid, check it")
+		}
+
+		activeOrderBook.EmitFilled(twinOrder.GetOrder())
+
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	s.EmitGridReady()
+
+	time.Sleep(2 * time.Second)
+	debugGrid(s.logger, s.grid, s.orderExecutor.ActiveMakerOrders())
+
+	bbgo.Sync(ctx, s)
+
+	return nil
+}
+
+func (s *Strategy) recoverEmptyGridOnTwinOrderBook(
+	ctx context.Context,
+	twinOrderBook *TwinOrderBook,
+	queryTradesService types.ExchangeTradeHistoryService,
+	queryOrderService types.ExchangeOrderQueryService,
+) error {
+	if twinOrderBook.EmptyTwinOrderSize() == 0 {
+		s.logger.Info("no empty grid")
+		return nil
+	}
+
+	existedOrders := twinOrderBook.SyncOrderMap()
+
+	until := time.Now()
+	since := until.Add(-1 * time.Hour)
+	// hard limit for recover
+	recoverSinceLimit := time.Date(2023, time.March, 10, 0, 0, 0, 0, time.UTC)
+
+	if s.RecoverGridWithin != 0 && until.Add(-1*s.RecoverGridWithin).After(recoverSinceLimit) {
+		recoverSinceLimit = until.Add(-1 * s.RecoverGridWithin)
+	}
+
+	for {
+		if err := queryTradesToUpdateTwinOrderBook(ctx, s.Symbol, twinOrderBook, queryTradesService, queryOrderService, existedOrders, since, until, s.debugLog); err != nil {
+			return errors.Wrapf(err, "failed to query trades to update twin orderbook")
+		}
+
+		until = since
+		since = until.Add(-6 * time.Hour)
+
+		if twinOrderBook.EmptyTwinOrderSize() == 0 {
+			s.logger.Infof("stop querying trades because there is no empty twin order on twin orderbook")
+			break
+		}
+
+		if s.GridProfitStats != nil && s.GridProfitStats.Since != nil && until.Before(*s.GridProfitStats.Since) {
+			s.logger.Infof("stop querying trades because the time range is out of the strategy's since (%s)", *s.GridProfitStats.Since)
+			break
+		}
+
+		if until.Before(recoverSinceLimit) {
+			s.logger.Infof("stop querying trades because the time range is out of the limit (%s)", recoverSinceLimit)
+			break
+		}
+	}
+
+	return nil
+}
+
 func buildTwinOrderBook(pins []Pin, orders []types.Order) (*TwinOrderBook, error) {
 	book := newTwinOrderBook(pins)
 
@@ -77,7 +270,8 @@ func queryTradesToUpdateTwinOrderBook(
 	queryOrderService types.ExchangeOrderQueryService,
 	existedOrders *types.SyncOrderMap,
 	since, until time.Time,
-	logger func(format string, args ...interface{})) error {
+	logger func(format string, args ...interface{}),
+) error {
 	if twinOrderBook == nil {
 		return fmt.Errorf("twin orderbook should not be nil, please check it")
 	}

From 40ca323b2dc726cfc1f5621063408f65205b857d Mon Sep 17 00:00:00 2001
From: chiahung <s199180626@gmail.com>
Date: Thu, 26 Oct 2023 16:29:05 +0800
Subject: [PATCH 24/42] merge recover logic

---
 pkg/strategy/grid2/recover.go | 43 +++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go
index e9be4866d..80a19008a 100644
--- a/pkg/strategy/grid2/recover.go
+++ b/pkg/strategy/grid2/recover.go
@@ -150,33 +150,36 @@ func (s *Strategy) recover(ctx context.Context) error {
 
 	s.logger.Infof("twin orderbook after adding open orders\n%s", activeOrdersInTwinOrderBook.String())
 
-	if err := s.recoverEmptyGridOnTwinOrderBook(ctx, activeOrdersInTwinOrderBook, historyService, s.orderQueryService); err != nil {
-		s.logger.WithError(err).Error("failed to recover empty grid")
-		return err
-	}
-
-	s.logger.Infof("twin orderbook after recovering\n%s", activeOrdersInTwinOrderBook.String())
-
-	if activeOrdersInTwinOrderBook.EmptyTwinOrderSize() > 0 {
-		return fmt.Errorf("there is still empty grid in twin orderbook")
-	}
-
-	for _, pin := range noTwinOrderPins {
-		twinOrder := activeOrdersInTwinOrderBook.GetTwinOrder(pin)
-		if twinOrder == nil {
-			return fmt.Errorf("should not get nil twin order after recovering empty grid, check it")
+	if len(noTwinOrderPins) != 0 {
+		if err := s.recoverEmptyGridOnTwinOrderBook(ctx, activeOrdersInTwinOrderBook, historyService, s.orderQueryService); err != nil {
+			s.logger.WithError(err).Error("failed to recover empty grid")
+			return err
 		}
 
-		if !twinOrder.Exist() {
-			return fmt.Errorf("should not get empty twin order after recovering empty grid, check it")
+		s.logger.Infof("twin orderbook after recovering no twin order on grid\n%s", activeOrdersInTwinOrderBook.String())
+
+		if activeOrdersInTwinOrderBook.EmptyTwinOrderSize() > 0 {
+			return fmt.Errorf("there is still empty grid in twin orderbook")
 		}
 
-		activeOrderBook.EmitFilled(twinOrder.GetOrder())
+		for _, pin := range noTwinOrderPins {
+			twinOrder := activeOrdersInTwinOrderBook.GetTwinOrder(pin)
+			if twinOrder == nil {
+				return fmt.Errorf("should not get nil twin order after recovering empty grid, check it")
+			}
 
-		time.Sleep(100 * time.Millisecond)
+			if !twinOrder.Exist() {
+				return fmt.Errorf("should not get empty twin order after recovering empty grid, check it")
+			}
+
+			activeOrderBook.EmitFilled(twinOrder.GetOrder())
+
+			time.Sleep(100 * time.Millisecond)
+		}
 	}
 
-	s.EmitGridReady()
+	// TODO: do not emit ready here, emit ready only once when opening grid or recovering grid after worker stopped
+	// s.EmitGridReady()
 
 	time.Sleep(2 * time.Second)
 	debugGrid(s.logger, s.grid, s.orderExecutor.ActiveMakerOrders())

From 2a85bbebf0e4540a76bf59e0b242087d0f778bf2 Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Fri, 27 Oct 2023 12:52:36 +0800
Subject: [PATCH 25/42] pkg/exchange: fix precision

---
 pkg/exchange/bybit/convert.go      | 4 ++--
 pkg/exchange/bybit/convert_test.go | 5 ++---
 pkg/exchange/kucoin/convert.go     | 4 ++--
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/pkg/exchange/bybit/convert.go b/pkg/exchange/bybit/convert.go
index b13032cd7..892493e33 100644
--- a/pkg/exchange/bybit/convert.go
+++ b/pkg/exchange/bybit/convert.go
@@ -16,8 +16,8 @@ func toGlobalMarket(m bybitapi.Instrument) types.Market {
 	return types.Market{
 		Symbol:          m.Symbol,
 		LocalSymbol:     m.Symbol,
-		PricePrecision:  int(math.Log10(m.LotSizeFilter.QuotePrecision.Float64())),
-		VolumePrecision: int(math.Log10(m.LotSizeFilter.BasePrecision.Float64())),
+		PricePrecision:  -int(math.Log10(m.LotSizeFilter.QuotePrecision.Float64())),
+		VolumePrecision: -int(math.Log10(m.LotSizeFilter.BasePrecision.Float64())),
 		QuoteCurrency:   m.QuoteCoin,
 		BaseCurrency:    m.BaseCoin,
 		MinNotional:     m.LotSizeFilter.MinOrderAmt,
diff --git a/pkg/exchange/bybit/convert_test.go b/pkg/exchange/bybit/convert_test.go
index 786234d8d..a5ddb08bc 100644
--- a/pkg/exchange/bybit/convert_test.go
+++ b/pkg/exchange/bybit/convert_test.go
@@ -2,7 +2,6 @@ package bybit
 
 import (
 	"fmt"
-	"math"
 	"strconv"
 	"testing"
 	"time"
@@ -67,8 +66,8 @@ func TestToGlobalMarket(t *testing.T) {
 	exp := types.Market{
 		Symbol:          inst.Symbol,
 		LocalSymbol:     inst.Symbol,
-		PricePrecision:  int(math.Log10(inst.LotSizeFilter.QuotePrecision.Float64())),
-		VolumePrecision: int(math.Log10(inst.LotSizeFilter.BasePrecision.Float64())),
+		PricePrecision:  8,
+		VolumePrecision: 6,
 		QuoteCurrency:   inst.QuoteCoin,
 		BaseCurrency:    inst.BaseCoin,
 		MinNotional:     inst.LotSizeFilter.MinOrderAmt,
diff --git a/pkg/exchange/kucoin/convert.go b/pkg/exchange/kucoin/convert.go
index d86f84db9..c33510522 100644
--- a/pkg/exchange/kucoin/convert.go
+++ b/pkg/exchange/kucoin/convert.go
@@ -39,8 +39,8 @@ func toGlobalMarket(m kucoinapi.Symbol) types.Market {
 	return types.Market{
 		Symbol:          symbol,
 		LocalSymbol:     m.Symbol,
-		PricePrecision:  int(math.Log10(m.PriceIncrement.Float64())), // convert 0.0001 to 4
-		VolumePrecision: int(math.Log10(m.BaseIncrement.Float64())),
+		PricePrecision:  -int(math.Log10(m.PriceIncrement.Float64())), // convert 0.0001 to 4
+		VolumePrecision: -int(math.Log10(m.BaseIncrement.Float64())),
 		QuoteCurrency:   m.QuoteCurrency,
 		BaseCurrency:    m.BaseCurrency,
 		MinNotional:     m.QuoteMinSize,

From e8c9801535d92004856681506bfad0f849b71123 Mon Sep 17 00:00:00 2001
From: narumi <toucans-cutouts0f@icloud.com>
Date: Fri, 27 Oct 2023 15:01:41 +0800
Subject: [PATCH 26/42] adjust quantity by max amount

---
 config/xalign.yaml              |  9 +++++++--
 pkg/strategy/xalign/strategy.go | 17 +++++++++++++++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/config/xalign.yaml b/config/xalign.yaml
index 51956f74f..6f07ba4b8 100644
--- a/config/xalign.yaml
+++ b/config/xalign.yaml
@@ -27,7 +27,6 @@ persistence:
     db: 0
 
 crossExchangeStrategies:
-
 - xalign:
     interval: 1m
     sessions:
@@ -41,4 +40,10 @@ crossExchangeStrategies:
       sell: [USDT]
     expectedBalances:
       BTC: 0.0440
-
+    useTakerOrder: false
+    dryRun: true
+    balanceToleranceRange: 10%
+    maxAmounts:
+      USDT: 100
+      USDC: 100
+      TWD: 3000
diff --git a/pkg/strategy/xalign/strategy.go b/pkg/strategy/xalign/strategy.go
index 140c1fb50..b6c4f49ad 100644
--- a/pkg/strategy/xalign/strategy.go
+++ b/pkg/strategy/xalign/strategy.go
@@ -45,6 +45,7 @@ type Strategy struct {
 	DryRun                   bool                        `json:"dryRun"`
 	BalanceToleranceRange    fixedpoint.Value            `json:"balanceToleranceRange"`
 	Duration                 types.Duration              `json:"for"`
+	MaxAmounts               map[string]fixedpoint.Value `json:"maxAmounts"`
 
 	faultBalanceRecords map[string][]TimeBalance
 
@@ -156,7 +157,7 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st
 			switch side {
 
 			case types.SideTypeBuy:
-				price := ticker.Sell
+				var price fixedpoint.Value
 				if taker {
 					price = ticker.Sell
 				} else if spread.Compare(market.TickSize) > 0 {
@@ -177,6 +178,12 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st
 					continue
 				}
 
+				maxAmount, ok := s.MaxAmounts[market.QuoteCurrency]
+				if ok {
+					requiredQuoteAmount = bbgo.AdjustQuantityByMaxAmount(requiredQuoteAmount, price, maxAmount)
+					log.Infof("adjusted quantity %f %s by max amount %f %s", requiredQuoteAmount.Float64(), market.BaseCurrency, maxAmount.Float64(), market.QuoteCurrency)
+				}
+
 				if quantity, ok := market.GreaterThanMinimalOrderQuantity(side, price, requiredQuoteAmount); ok {
 					return session, &types.SubmitOrder{
 						Symbol:      symbol,
@@ -190,7 +197,7 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st
 				}
 
 			case types.SideTypeSell:
-				price := ticker.Buy
+				var price fixedpoint.Value
 				if taker {
 					price = ticker.Buy
 				} else if spread.Compare(market.TickSize) > 0 {
@@ -209,6 +216,12 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st
 					continue
 				}
 
+				maxAmount, ok := s.MaxAmounts[market.QuoteCurrency]
+				if ok {
+					q = bbgo.AdjustQuantityByMaxAmount(q, price, maxAmount)
+					log.Infof("adjusted quantity %f %s by max amount %f %s", q.Float64(), market.BaseCurrency, maxAmount.Float64(), market.QuoteCurrency)
+				}
+
 				if quantity, ok := market.GreaterThanMinimalOrderQuantity(side, price, q); ok {
 					return session, &types.SubmitOrder{
 						Symbol:      symbol,

From ba7e26c800b4fd9a9b20b172b46e316c5a8dbbec Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Fri, 27 Oct 2023 15:28:35 +0800
Subject: [PATCH 27/42] pkg/exchange: use NumFractionalDigits instead of
 math.Log10(Float64) due to precision problem

---
 pkg/exchange/bybit/convert.go  | 5 ++---
 pkg/exchange/kucoin/convert.go | 5 ++---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/pkg/exchange/bybit/convert.go b/pkg/exchange/bybit/convert.go
index 892493e33..d89f194fe 100644
--- a/pkg/exchange/bybit/convert.go
+++ b/pkg/exchange/bybit/convert.go
@@ -2,7 +2,6 @@ package bybit
 
 import (
 	"fmt"
-	"math"
 	"strconv"
 	"time"
 
@@ -16,8 +15,8 @@ func toGlobalMarket(m bybitapi.Instrument) types.Market {
 	return types.Market{
 		Symbol:          m.Symbol,
 		LocalSymbol:     m.Symbol,
-		PricePrecision:  -int(math.Log10(m.LotSizeFilter.QuotePrecision.Float64())),
-		VolumePrecision: -int(math.Log10(m.LotSizeFilter.BasePrecision.Float64())),
+		PricePrecision:  m.LotSizeFilter.QuotePrecision.NumFractionalDigits(),
+		VolumePrecision: m.LotSizeFilter.BasePrecision.NumFractionalDigits(),
 		QuoteCurrency:   m.QuoteCoin,
 		BaseCurrency:    m.BaseCoin,
 		MinNotional:     m.LotSizeFilter.MinOrderAmt,
diff --git a/pkg/exchange/kucoin/convert.go b/pkg/exchange/kucoin/convert.go
index c33510522..e83ade5d3 100644
--- a/pkg/exchange/kucoin/convert.go
+++ b/pkg/exchange/kucoin/convert.go
@@ -3,7 +3,6 @@ package kucoin
 import (
 	"fmt"
 	"hash/fnv"
-	"math"
 	"strings"
 	"time"
 
@@ -39,8 +38,8 @@ func toGlobalMarket(m kucoinapi.Symbol) types.Market {
 	return types.Market{
 		Symbol:          symbol,
 		LocalSymbol:     m.Symbol,
-		PricePrecision:  -int(math.Log10(m.PriceIncrement.Float64())), // convert 0.0001 to 4
-		VolumePrecision: -int(math.Log10(m.BaseIncrement.Float64())),
+		PricePrecision:  m.PriceIncrement.NumFractionalDigits(), // convert 0.0001 to 4
+		VolumePrecision: m.BaseIncrement.NumFractionalDigits(),
 		QuoteCurrency:   m.QuoteCurrency,
 		BaseCurrency:    m.BaseCurrency,
 		MinNotional:     m.QuoteMinSize,

From d07b7669390e4daf9c6e7078257b15b2007936ff Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Fri, 27 Oct 2023 16:03:03 +0800
Subject: [PATCH 28/42] pkg/exchange: Use the same conn to avoid concurrent
 write issues.

---
 pkg/exchange/bybit/stream.go | 48 +++++++++---------------------------
 pkg/types/stream.go          | 19 ++++++++------
 2 files changed, 22 insertions(+), 45 deletions(-)

diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go
index c6b42cb9b..eb4137ed3 100644
--- a/pkg/exchange/bybit/stream.go
+++ b/pkg/exchange/bybit/stream.go
@@ -15,10 +15,6 @@ import (
 )
 
 const (
-	// Bybit: To avoid network or program issues, we recommend that you send the ping heartbeat packet every 20 seconds
-	// to maintain the WebSocket connection.
-	pingInterval = 20 * time.Second
-
 	// spotArgsLimit can input up to 10 args for each subscription request sent to one connection.
 	spotArgsLimit = 10
 )
@@ -244,40 +240,18 @@ func (s *Stream) parseWebSocketEvent(in []byte) (interface{}, error) {
 }
 
 // ping implements the Bybit text message of WebSocket PingPong.
-func (s *Stream) ping(ctx context.Context, conn *websocket.Conn, cancelFunc context.CancelFunc) {
-	defer func() {
-		log.Debug("[bybit] ping worker stopped")
-		cancelFunc()
-	}()
-
-	var pingTicker = time.NewTicker(pingInterval)
-	defer pingTicker.Stop()
-
-	for {
-		select {
-
-		case <-ctx.Done():
-			return
-
-		case <-s.CloseC:
-			return
-
-		case <-pingTicker.C:
-			// it's just for maintaining the liveliness of the connection, so comment out ReqId.
-			err := conn.WriteJSON(struct {
-				//ReqId string `json:"req_id"`
-				Op WsOpType `json:"op"`
-			}{
-				//ReqId: uuid.NewString(),
-				Op: WsOpTypePing,
-			})
-			if err != nil {
-				log.WithError(err).Error("ping error", err)
-				s.Reconnect()
-				return
-			}
-		}
+func (s *Stream) ping(conn *websocket.Conn) error {
+	err := conn.WriteJSON(struct {
+		Op WsOpType `json:"op"`
+	}{
+		Op: WsOpTypePing,
+	})
+	if err != nil {
+		log.WithError(err).Error("ping error")
+		return err
 	}
+
+	return nil
 }
 
 func (s *Stream) handlerConnect() {
diff --git a/pkg/types/stream.go b/pkg/types/stream.go
index f65927b61..4ce8c161f 100644
--- a/pkg/types/stream.go
+++ b/pkg/types/stream.go
@@ -57,8 +57,8 @@ type Parser func(message []byte) (interface{}, error)
 
 type Dispatcher func(e interface{})
 
-// HeartBeat keeps connection alive by sending the heartbeat packet.
-type HeartBeat func(ctxConn context.Context, conn *websocket.Conn, cancelConn context.CancelFunc)
+// HeartBeat keeps connection alive by sending the ping packet.
+type HeartBeat func(conn *websocket.Conn) error
 
 type BeforeConnect func(ctx context.Context) error
 
@@ -86,7 +86,7 @@ type StandardStream struct {
 
 	// sg is used to wait until the previous routines are closed.
 	// only handle routines used internally, avoid including external callback func to prevent issues if they have
-	// bugs and cannot terminate. e.q. heartBeat
+	// bugs and cannot terminate.
 	sg SyncGroup
 
 	// ReconnectC is a signal channel for reconnecting
@@ -319,6 +319,14 @@ func (s *StandardStream) ping(
 			return
 
 		case <-pingTicker.C:
+			if s.heartBeat != nil {
+				if err := s.heartBeat(conn); err != nil {
+					// log errors at the concrete class so that we can identify which exchange encountered an error
+					s.Reconnect()
+					return
+				}
+			}
+
 			if err := conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(writeTimeout)); err != nil {
 				log.WithError(err).Error("ping error", err)
 				s.Reconnect()
@@ -432,11 +440,6 @@ func (s *StandardStream) DialAndConnect(ctx context.Context) error {
 		s.ping(connCtx, conn, connCancel, pingInterval)
 	})
 	s.sg.Run()
-
-	if s.heartBeat != nil {
-		// not included in wg, as it is an external callback func.
-		go s.heartBeat(connCtx, conn, connCancel)
-	}
 	return nil
 }
 

From 39c3d23da323e3222df5d610695a8a776b92f5f0 Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Tue, 24 Oct 2023 22:17:08 +0800
Subject: [PATCH 29/42] pkg/exchange: support ping/pong

---
 pkg/exchange/bitget/stream.go | 43 +++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go
index eadf25b48..5a2f6f23d 100644
--- a/pkg/exchange/bitget/stream.go
+++ b/pkg/exchange/bitget/stream.go
@@ -1,12 +1,26 @@
 package bitget
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"time"
 
 	"github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi"
 	"github.com/c9s/bbgo/pkg/types"
+	"github.com/gorilla/websocket"
+)
+
+const (
+	// Client should keep ping the server in every 30 seconds. Server will close the connections which has no ping over
+	// 120 seconds(even when the client is still receiving data from the server)
+	pingInterval = 30 * time.Second
+)
+
+var (
+	pingBytes = []byte("ping")
+	pongBytes = []byte("pong")
 )
 
 //go:generate callbackgen -type Stream
@@ -25,6 +39,7 @@ func NewStream() *Stream {
 	stream.SetEndpointCreator(stream.createEndpoint)
 	stream.SetParser(parseWebSocketEvent)
 	stream.SetDispatcher(stream.dispatchEvent)
+	stream.SetHeartBeat(stream.ping)
 	stream.OnConnect(stream.handlerConnect)
 
 	stream.OnBookEvent(stream.handleBookEvent)
@@ -92,6 +107,12 @@ func (s *Stream) dispatchEvent(event interface{}) {
 
 	case *MarketTradeEvent:
 		s.EmitMarketTradeEvent(*e)
+
+	case []byte:
+		// We only handle the 'pong' case. Others are unexpected.
+		if !bytes.Equal(e, pongBytes) {
+			log.Errorf("invalid event: %q", e)
+		}
 	}
 }
 
@@ -116,6 +137,16 @@ func (s *Stream) handleBookEvent(o BookEvent) {
 	}
 }
 
+// ping implements the bitget text message of WebSocket PingPong.
+func (s *Stream) ping(conn *websocket.Conn) error {
+	err := conn.WriteMessage(websocket.TextMessage, pingBytes)
+	if err != nil {
+		log.WithError(err).Error("ping error", err)
+		return nil
+	}
+	return nil
+}
+
 func convertSubscription(sub types.Subscription) (WsArg, error) {
 	arg := WsArg{
 		// support spot only
@@ -146,6 +177,18 @@ func convertSubscription(sub types.Subscription) (WsArg, error) {
 }
 
 func parseWebSocketEvent(in []byte) (interface{}, error) {
+	switch {
+	case bytes.Equal(in, pongBytes):
+		// Return the original raw data may seem redundant because we can validate the string and return nil,
+		// but we cannot return nil to a lower level handler. This can cause confusion in the next handler, such as
+		// the dispatch handler. Therefore, I return the original raw data.
+		return in, nil
+	default:
+		return parseEvent(in)
+	}
+}
+
+func parseEvent(in []byte) (interface{}, error) {
 	var event WsEvent
 
 	err := json.Unmarshal(in, &event)

From 671772a76775f9321e7e098d00031fecbfdcfb09 Mon Sep 17 00:00:00 2001
From: chiahung <s199180626@gmail.com>
Date: Mon, 30 Oct 2023 16:28:34 +0800
Subject: [PATCH 30/42] FIX: retry to get open orders only for 5 times and do
 not sync orders updated in 3 min

---
 pkg/exchange/retry/order.go                | 19 +++++++++++++++++++
 pkg/strategy/grid2/active_order_recover.go | 15 +++++++++++----
 pkg/strategy/grid2/recover.go              |  2 +-
 3 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/pkg/exchange/retry/order.go b/pkg/exchange/retry/order.go
index 15b59c6af..5a8c8855b 100644
--- a/pkg/exchange/retry/order.go
+++ b/pkg/exchange/retry/order.go
@@ -47,6 +47,15 @@ func GeneralBackoff(ctx context.Context, op backoff2.Operation) (err error) {
 	return err
 }
 
+func GeneralBackoffLite(ctx context.Context, op backoff2.Operation) (err error) {
+	err = backoff2.Retry(op, backoff2.WithContext(
+		backoff2.WithMaxRetries(
+			backoff2.NewExponentialBackOff(),
+			5),
+		ctx))
+	return err
+}
+
 func QueryOpenOrdersUntilSuccessful(ctx context.Context, ex types.Exchange, symbol string) (openOrders []types.Order, err error) {
 	var op = func() (err2 error) {
 		openOrders, err2 = ex.QueryOpenOrders(ctx, symbol)
@@ -57,6 +66,16 @@ func QueryOpenOrdersUntilSuccessful(ctx context.Context, ex types.Exchange, symb
 	return openOrders, err
 }
 
+func QueryOpenOrdersUntilSuccessfulLite(ctx context.Context, ex types.Exchange, symbol string) (openOrders []types.Order, err error) {
+	var op = func() (err2 error) {
+		openOrders, err2 = ex.QueryOpenOrders(ctx, symbol)
+		return err2
+	}
+
+	err = GeneralBackoffLite(ctx, op)
+	return openOrders, err
+}
+
 func QueryOrderUntilSuccessful(ctx context.Context, query types.ExchangeOrderQueryService, opts types.OrderQuery) (order *types.Order, err error) {
 	var op = func() (err2 error) {
 		order, err2 = query.QueryOrder(ctx, opts)
diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go
index 5042084fc..82269832e 100644
--- a/pkg/strategy/grid2/active_order_recover.go
+++ b/pkg/strategy/grid2/active_order_recover.go
@@ -89,9 +89,10 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) {
 func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error {
 	opts.logger.Infof("[ActiveOrderRecover] syncActiveOrders")
 
-	notAddNonExistingOpenOrdersAfter := time.Now().Add(-5 * time.Minute)
+	// do not sync orders which is updated in 3 min, because we may receive from websocket and handle it twice
+	doNotSyncAfter := time.Now().Add(-3 * time.Minute)
 
-	openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, opts.exchange, opts.activeOrderBook.Symbol)
+	openOrders, err := retry.QueryOpenOrdersUntilSuccessfulLite(ctx, opts.exchange, opts.activeOrderBook.Symbol)
 	if err != nil {
 		opts.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time")
 		return errors.Wrapf(err, "[ActiveOrderRecover] failed to query open orders, skip this time")
@@ -116,6 +117,10 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error {
 			delete(openOrdersMap, activeOrder.OrderID)
 		} else {
 			opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID)
+			if activeOrder.UpdateTime.After(doNotSyncAfter) {
+				opts.logger.Infof("active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID)
+				continue
+			}
 
 			// sleep 100ms to avoid DDOS
 			time.Sleep(100 * time.Millisecond)
@@ -130,8 +135,10 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error {
 
 	// update open orders not in active orders
 	for _, openOrder := range openOrdersMap {
-		// we don't add open orders into active orderbook if updated in 5 min
-		if openOrder.UpdateTime.After(notAddNonExistingOpenOrdersAfter) {
+		opts.logger.Infof("found open order #%d is not in active orderbook, updating...", openOrder.OrderID)
+		// we don't add open orders into active orderbook if updated in 3 min, because we may receive message from websocket and add it twice.
+		if openOrder.UpdateTime.After(doNotSyncAfter) {
+			opts.logger.Infof("open order #%d is updated in 3 min, skip updating...", openOrder.OrderID)
 			continue
 		}
 
diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go
index 80a19008a..0634b72d4 100644
--- a/pkg/strategy/grid2/recover.go
+++ b/pkg/strategy/grid2/recover.go
@@ -54,7 +54,7 @@ func (s *Strategy) recover(ctx context.Context) error {
 	activeOrderBook := s.orderExecutor.ActiveMakerOrders()
 	activeOrders := activeOrderBook.Orders()
 
-	openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol)
+	openOrders, err := retry.QueryOpenOrdersUntilSuccessfulLite(ctx, s.session.Exchange, s.Symbol)
 	if err != nil {
 		return err
 	}

From d33240ec83781ed92010941b7fc3104e38e3d3fb Mon Sep 17 00:00:00 2001
From: chiahung <s199180626@gmail.com>
Date: Mon, 30 Oct 2023 17:17:36 +0800
Subject: [PATCH 31/42] rename and simplify import

---
 pkg/exchange/retry/order.go                | 26 +++++++++++-----------
 pkg/strategy/grid2/active_order_recover.go |  8 +++----
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/pkg/exchange/retry/order.go b/pkg/exchange/retry/order.go
index 5a8c8855b..df6ef6b59 100644
--- a/pkg/exchange/retry/order.go
+++ b/pkg/exchange/retry/order.go
@@ -5,10 +5,9 @@ import (
 	"errors"
 	"strconv"
 
-	backoff2 "github.com/cenkalti/backoff/v4"
+	"github.com/cenkalti/backoff/v4"
 
 	"github.com/c9s/bbgo/pkg/types"
-	"github.com/c9s/bbgo/pkg/util/backoff"
 )
 
 type advancedOrderCancelService interface {
@@ -18,7 +17,7 @@ type advancedOrderCancelService interface {
 }
 
 func QueryOrderUntilFilled(ctx context.Context, queryOrderService types.ExchangeOrderQueryService, symbol string, orderId uint64) (o *types.Order, err error) {
-	err = backoff.RetryGeneral(ctx, func() (err2 error) {
+	var op = func() (err2 error) {
 		o, err2 = queryOrderService.QueryOrder(ctx, types.OrderQuery{
 			Symbol:  symbol,
 			OrderID: strconv.FormatUint(orderId, 10),
@@ -33,24 +32,25 @@ func QueryOrderUntilFilled(ctx context.Context, queryOrderService types.Exchange
 		}
 
 		return err2
-	})
+	}
 
+	err = GeneralBackoff(ctx, op)
 	return o, err
 }
 
-func GeneralBackoff(ctx context.Context, op backoff2.Operation) (err error) {
-	err = backoff2.Retry(op, backoff2.WithContext(
-		backoff2.WithMaxRetries(
-			backoff2.NewExponentialBackOff(),
+func GeneralBackoff(ctx context.Context, op backoff.Operation) (err error) {
+	err = backoff.Retry(op, backoff.WithContext(
+		backoff.WithMaxRetries(
+			backoff.NewExponentialBackOff(),
 			101),
 		ctx))
 	return err
 }
 
-func GeneralBackoffLite(ctx context.Context, op backoff2.Operation) (err error) {
-	err = backoff2.Retry(op, backoff2.WithContext(
-		backoff2.WithMaxRetries(
-			backoff2.NewExponentialBackOff(),
+func GeneralLiteBackoff(ctx context.Context, op backoff.Operation) (err error) {
+	err = backoff.Retry(op, backoff.WithContext(
+		backoff.WithMaxRetries(
+			backoff.NewExponentialBackOff(),
 			5),
 		ctx))
 	return err
@@ -72,7 +72,7 @@ func QueryOpenOrdersUntilSuccessfulLite(ctx context.Context, ex types.Exchange,
 		return err2
 	}
 
-	err = GeneralBackoffLite(ctx, op)
+	err = GeneralLiteBackoff(ctx, op)
 	return openOrders, err
 }
 
diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go
index 82269832e..cfdaeac80 100644
--- a/pkg/strategy/grid2/active_order_recover.go
+++ b/pkg/strategy/grid2/active_order_recover.go
@@ -89,8 +89,8 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) {
 func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error {
 	opts.logger.Infof("[ActiveOrderRecover] syncActiveOrders")
 
-	// do not sync orders which is updated in 3 min, because we may receive from websocket and handle it twice
-	doNotSyncAfter := time.Now().Add(-3 * time.Minute)
+	// only sync orders which is updated over 3 min, because we may receive from websocket and handle it twice
+	syncBefore := time.Now().Add(-3 * time.Minute)
 
 	openOrders, err := retry.QueryOpenOrdersUntilSuccessfulLite(ctx, opts.exchange, opts.activeOrderBook.Symbol)
 	if err != nil {
@@ -117,7 +117,7 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error {
 			delete(openOrdersMap, activeOrder.OrderID)
 		} else {
 			opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID)
-			if activeOrder.UpdateTime.After(doNotSyncAfter) {
+			if activeOrder.UpdateTime.After(syncBefore) {
 				opts.logger.Infof("active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID)
 				continue
 			}
@@ -137,7 +137,7 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error {
 	for _, openOrder := range openOrdersMap {
 		opts.logger.Infof("found open order #%d is not in active orderbook, updating...", openOrder.OrderID)
 		// we don't add open orders into active orderbook if updated in 3 min, because we may receive message from websocket and add it twice.
-		if openOrder.UpdateTime.After(doNotSyncAfter) {
+		if openOrder.UpdateTime.After(syncBefore) {
 			opts.logger.Infof("open order #%d is updated in 3 min, skip updating...", openOrder.OrderID)
 			continue
 		}

From b8401ee177845e70ecf18c17509ce1a55329aa71 Mon Sep 17 00:00:00 2001
From: Himanshu Kumar Mahto <93067059+HimanshuMahto@users.noreply.github.com>
Date: Tue, 31 Oct 2023 01:51:04 +0530
Subject: [PATCH 32/42] grammatical error in the code_of_conduct file

---
 CODE_OF_CONDUCT.md | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index b70133c9c..e5e203c08 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -2,7 +2,7 @@
 
 ## Our Pledge
 
-We as members, contributors, and leaders pledge to make participation in our
+We as members, contributors, and leaders pledge to participate in our
 community a harassment-free experience for everyone, regardless of age, body
 size, visible or invisible disability, ethnicity, sex characteristics, gender
 identity and expression, level of experience, education, socio-economic status,
@@ -15,7 +15,7 @@ diverse, inclusive, and healthy community.
 ## Our Standards
 
 Examples of behavior that contributes to a positive environment for our
-community include:
+community includes:
 
 * Demonstrating empathy and kindness toward other people
 * Being respectful of differing opinions, viewpoints, and experiences
@@ -33,7 +33,7 @@ Examples of unacceptable behavior include:
 * Public or private harassment
 * Publishing others' private information, such as a physical or email
   address, without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
+* Other conduct that could reasonably be considered inappropriate in a
   professional setting
 
 ## Enforcement Responsibilities
@@ -50,7 +50,7 @@ decisions when appropriate.
 
 ## Scope
 
-This Code of Conduct applies within all community spaces, and also applies when
+This Code of Conduct applies within all community spaces and also applies when
 an individual is officially representing the community in public spaces.
 Examples of representing our community include using an official e-mail address,
 posting via an official social media account, or acting as an appointed
@@ -82,12 +82,11 @@ behavior was inappropriate. A public apology may be requested.
 
 ### 2. Warning
 
-**Community Impact**: A violation through a single incident or series
-of actions.
+**Community Impact**: This violation occurs through a single incident or a series of actions.
 
 **Consequence**: A warning with consequences for continued behavior. No
 interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
+those enforcing the Code of Conduct, for a specified period. This
 includes avoiding interactions in community spaces as well as external channels
 like social media. Violating these terms may lead to a temporary or
 permanent ban.
@@ -98,7 +97,7 @@ permanent ban.
 sustained inappropriate behavior.
 
 **Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
+communication with the community for a specified period. No public or
 private interaction with the people involved, including unsolicited interaction
 with those enforcing the Code of Conduct, is allowed during this period.
 Violating these terms may lead to a permanent ban.

From 7c19bb9e20fba4eab7f6306088c28a8d7c836061 Mon Sep 17 00:00:00 2001
From: narumi <toucans-cutouts0f@icloud.com>
Date: Tue, 31 Oct 2023 13:53:12 +0800
Subject: [PATCH 33/42] submit one order at a time

---
 config/rebalance.yaml                         |  10 +-
 .../rebalance/multi_market_strategy.go        |  44 ++++
 pkg/strategy/rebalance/position_map.go        |  12 +-
 pkg/strategy/rebalance/profit_stats_map.go    |  14 +-
 pkg/strategy/rebalance/strategy.go            | 245 +++++++-----------
 5 files changed, 158 insertions(+), 167 deletions(-)
 create mode 100644 pkg/strategy/rebalance/multi_market_strategy.go

diff --git a/config/rebalance.yaml b/config/rebalance.yaml
index 14a784c93..bdcd5f6f5 100644
--- a/config/rebalance.yaml
+++ b/config/rebalance.yaml
@@ -12,9 +12,9 @@ backtest:
   startTime: "2022-01-01"
   endTime: "2022-10-01"
   symbols:
-  - BTCUSDT
-  - ETHUSDT
-  - MAXUSDT
+    - BTCUSDT
+    - ETHUSDT
+    - MAXUSDT
   account:
     max:
       makerFeeRate: 0.075%
@@ -28,7 +28,7 @@ backtest:
 exchangeStrategies:
   - on: max
     rebalance:
-      interval: 1d
+      cronExpression: "@every 1s"
       quoteCurrency: USDT
       targetWeights:
         BTC: 50%
@@ -37,5 +37,5 @@ exchangeStrategies:
       threshold: 1%
       maxAmount: 1_000 # max amount to buy or sell per order
       orderType: LIMIT_MAKER # LIMIT, LIMIT_MAKER or MARKET
-      dryRun: false
+      dryRun: true
       onStart: true
diff --git a/pkg/strategy/rebalance/multi_market_strategy.go b/pkg/strategy/rebalance/multi_market_strategy.go
new file mode 100644
index 000000000..e7d17dd0b
--- /dev/null
+++ b/pkg/strategy/rebalance/multi_market_strategy.go
@@ -0,0 +1,44 @@
+package rebalance
+
+import (
+	"context"
+
+	"github.com/c9s/bbgo/pkg/bbgo"
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+type MultiMarketStrategy struct {
+	Environ *bbgo.Environment
+	Session *bbgo.ExchangeSession
+
+	PositionMap      PositionMap    `persistence:"positionMap"`
+	ProfitStatsMap   ProfitStatsMap `persistence:"profitStatsMap"`
+	OrderExecutorMap GeneralOrderExecutorMap
+
+	parent, ctx context.Context
+	cancel      context.CancelFunc
+}
+
+func (s *MultiMarketStrategy) Initialize(ctx context.Context, environ *bbgo.Environment, session *bbgo.ExchangeSession, markets map[string]types.Market, strategyID string) {
+	s.parent = ctx
+	s.ctx, s.cancel = context.WithCancel(ctx)
+
+	s.Environ = environ
+	s.Session = session
+
+	if s.PositionMap == nil {
+		s.PositionMap = make(PositionMap)
+	}
+	s.PositionMap.CreatePositions(markets)
+
+	if s.ProfitStatsMap == nil {
+		s.ProfitStatsMap = make(ProfitStatsMap)
+	}
+	s.ProfitStatsMap.CreateProfitStats(markets)
+
+	s.OrderExecutorMap = NewGeneralOrderExecutorMap(session, s.PositionMap)
+	s.OrderExecutorMap.BindEnvironment(environ)
+	s.OrderExecutorMap.BindProfitStats(s.ProfitStatsMap)
+	s.OrderExecutorMap.Sync(ctx, s)
+	s.OrderExecutorMap.Bind()
+}
diff --git a/pkg/strategy/rebalance/position_map.go b/pkg/strategy/rebalance/position_map.go
index 772d1726c..73bdda499 100644
--- a/pkg/strategy/rebalance/position_map.go
+++ b/pkg/strategy/rebalance/position_map.go
@@ -6,17 +6,17 @@ import (
 
 type PositionMap map[string]*types.Position
 
-func (m PositionMap) CreatePositions(markets []types.Market) PositionMap {
-	for _, market := range markets {
-		if _, ok := m[market.Symbol]; ok {
+func (m PositionMap) CreatePositions(markets map[string]types.Market) PositionMap {
+	for symbol, market := range markets {
+		if _, ok := m[symbol]; ok {
 			continue
 		}
 
-		log.Infof("creating position for symbol %s", market.Symbol)
+		log.Infof("creating position for symbol %s", symbol)
 		position := types.NewPositionFromMarket(market)
 		position.Strategy = ID
-		position.StrategyInstanceID = instanceID(market.Symbol)
-		m[market.Symbol] = position
+		position.StrategyInstanceID = instanceID(symbol)
+		m[symbol] = position
 	}
 	return m
 }
diff --git a/pkg/strategy/rebalance/profit_stats_map.go b/pkg/strategy/rebalance/profit_stats_map.go
index a84bf5cc9..29e427a6e 100644
--- a/pkg/strategy/rebalance/profit_stats_map.go
+++ b/pkg/strategy/rebalance/profit_stats_map.go
@@ -1,17 +1,19 @@
 package rebalance
 
-import "github.com/c9s/bbgo/pkg/types"
+import (
+	"github.com/c9s/bbgo/pkg/types"
+)
 
 type ProfitStatsMap map[string]*types.ProfitStats
 
-func (m ProfitStatsMap) CreateProfitStats(markets []types.Market) ProfitStatsMap {
-	for _, market := range markets {
-		if _, ok := m[market.Symbol]; ok {
+func (m ProfitStatsMap) CreateProfitStats(markets map[string]types.Market) ProfitStatsMap {
+	for symbol, market := range markets {
+		if _, ok := m[symbol]; ok {
 			continue
 		}
 
-		log.Infof("creating profit stats for symbol %s", market.Symbol)
-		m[market.Symbol] = types.NewProfitStats(market)
+		log.Infof("creating profit stats for symbol %s", symbol)
+		m[symbol] = types.NewProfitStats(market)
 	}
 	return m
 }
diff --git a/pkg/strategy/rebalance/strategy.go b/pkg/strategy/rebalance/strategy.go
index e15e2507f..22896d24b 100644
--- a/pkg/strategy/rebalance/strategy.go
+++ b/pkg/strategy/rebalance/strategy.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/robfig/cron/v3"
 	"github.com/sirupsen/logrus"
 
 	"github.com/c9s/bbgo/pkg/bbgo"
@@ -15,6 +16,7 @@ import (
 const ID = "rebalance"
 
 var log = logrus.WithField("strategy", ID)
+var two = fixedpoint.NewFromFloat(2.0)
 
 func init() {
 	bbgo.RegisterStrategy(ID, &Strategy{})
@@ -25,23 +27,24 @@ func instanceID(symbol string) string {
 }
 
 type Strategy struct {
+	*MultiMarketStrategy
+
 	Environment *bbgo.Environment
 
-	Interval      types.Interval   `json:"interval"`
-	QuoteCurrency string           `json:"quoteCurrency"`
-	TargetWeights types.ValueMap   `json:"targetWeights"`
-	Threshold     fixedpoint.Value `json:"threshold"`
-	MaxAmount     fixedpoint.Value `json:"maxAmount"` // max amount to buy or sell per order
-	OrderType     types.OrderType  `json:"orderType"`
-	DryRun        bool             `json:"dryRun"`
-	OnStart       bool             `json:"onStart"` // rebalance on start
+	CronExpression string           `json:"cronExpression"`
+	QuoteCurrency  string           `json:"quoteCurrency"`
+	TargetWeights  types.ValueMap   `json:"targetWeights"`
+	Threshold      fixedpoint.Value `json:"threshold"`
+	MaxAmount      fixedpoint.Value `json:"maxAmount"` // max amount to buy or sell per order
+	OrderType      types.OrderType  `json:"orderType"`
+	DryRun         bool             `json:"dryRun"`
+	OnStart        bool             `json:"onStart"` // rebalance on start
 
-	PositionMap    PositionMap    `persistence:"positionMap"`
-	ProfitStatsMap ProfitStatsMap `persistence:"profitStatsMap"`
-
-	session          *bbgo.ExchangeSession
-	orderExecutorMap GeneralOrderExecutorMap
-	activeOrderBook  *bbgo.ActiveOrderBook
+	session         *bbgo.ExchangeSession
+	symbols         []string
+	markets         map[string]types.Market
+	activeOrderBook *bbgo.ActiveOrderBook
+	cron            *cron.Cron
 }
 
 func (s *Strategy) Defaults() error {
@@ -52,6 +55,13 @@ func (s *Strategy) Defaults() error {
 }
 
 func (s *Strategy) Initialize() error {
+	for currency := range s.TargetWeights {
+		if currency == s.QuoteCurrency {
+			continue
+		}
+
+		s.symbols = append(s.symbols, currency+s.QuoteCurrency)
+	}
 	return nil
 }
 
@@ -84,35 +94,22 @@ func (s *Strategy) Validate() error {
 	return nil
 }
 
-func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) {
-	for _, symbol := range s.symbols() {
-		session.Subscribe(types.KLineChannel, symbol, types.SubscribeOptions{Interval: s.Interval})
-	}
-}
+func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) {}
 
 func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error {
 	s.session = session
 
-	markets, err := s.markets()
-	if err != nil {
-		return err
+	s.markets = make(map[string]types.Market)
+	for _, symbol := range s.symbols {
+		market, ok := s.session.Market(symbol)
+		if !ok {
+			return fmt.Errorf("market %s not found", symbol)
+		}
+		s.markets[symbol] = market
 	}
 
-	if s.PositionMap == nil {
-		s.PositionMap = make(PositionMap)
-	}
-	s.PositionMap.CreatePositions(markets)
-
-	if s.ProfitStatsMap == nil {
-		s.ProfitStatsMap = make(ProfitStatsMap)
-	}
-	s.ProfitStatsMap.CreateProfitStats(markets)
-
-	s.orderExecutorMap = NewGeneralOrderExecutorMap(session, s.PositionMap)
-	s.orderExecutorMap.BindEnvironment(s.Environment)
-	s.orderExecutorMap.BindProfitStats(s.ProfitStatsMap)
-	s.orderExecutorMap.Bind()
-	s.orderExecutorMap.Sync(ctx, s)
+	s.MultiMarketStrategy = &MultiMarketStrategy{}
+	s.MultiMarketStrategy.Initialize(ctx, s.Environment, session, s.markets, ID)
 
 	s.activeOrderBook = bbgo.NewActiveOrderBook("")
 	s.activeOrderBook.BindStream(s.session.UserDataStream)
@@ -123,16 +120,18 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.
 		}
 	})
 
-	s.session.MarketDataStream.OnKLineClosed(func(kline types.KLine) {
-		s.rebalance(ctx)
-	})
-
 	// the shutdown handler, you can cancel all orders
 	bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) {
 		defer wg.Done()
-		_ = s.orderExecutorMap.GracefulCancel(ctx)
+		_ = s.OrderExecutorMap.GracefulCancel(ctx)
 	})
 
+	s.cron = cron.New()
+	s.cron.AddFunc(s.CronExpression, func() {
+		s.rebalance(ctx)
+	})
+	s.cron.Start()
+
 	return nil
 }
 
@@ -142,21 +141,24 @@ func (s *Strategy) rebalance(ctx context.Context) {
 		log.WithError(err).Errorf("failed to cancel orders")
 	}
 
-	submitOrders, err := s.generateSubmitOrders(ctx)
+	order, err := s.generateOrder(ctx)
 	if err != nil {
-		log.WithError(err).Error("failed to generate submit orders")
+		log.WithError(err).Error("failed to generate order")
 		return
 	}
-	for _, order := range submitOrders {
-		log.Infof("generated submit order: %s", order.String())
+
+	if order == nil {
+		log.Info("no order generated")
+		return
 	}
+	log.Infof("generated order: %s", order.String())
 
 	if s.DryRun {
 		log.Infof("dry run, not submitting orders")
 		return
 	}
 
-	createdOrders, err := s.orderExecutorMap.SubmitOrders(ctx, submitOrders...)
+	createdOrders, err := s.OrderExecutorMap.SubmitOrders(ctx, *order)
 	if err != nil {
 		log.WithError(err).Error("failed to submit orders")
 		return
@@ -164,7 +166,7 @@ func (s *Strategy) rebalance(ctx context.Context) {
 	s.activeOrderBook.Add(createdOrders...)
 }
 
-func (s *Strategy) prices(ctx context.Context) (types.ValueMap, error) {
+func (s *Strategy) queryMidPrices(ctx context.Context) (types.ValueMap, error) {
 	m := make(types.ValueMap)
 	for currency := range s.TargetWeights {
 		if currency == s.QuoteCurrency {
@@ -177,12 +179,12 @@ func (s *Strategy) prices(ctx context.Context) (types.ValueMap, error) {
 			return nil, err
 		}
 
-		m[currency] = ticker.Buy.Add(ticker.Sell).Div(fixedpoint.NewFromFloat(2.0))
+		m[currency] = ticker.Buy.Add(ticker.Sell).Div(two)
 	}
 	return m, nil
 }
 
-func (s *Strategy) balances() (types.BalanceMap, error) {
+func (s *Strategy) selectBalances() (types.BalanceMap, error) {
 	m := make(types.BalanceMap)
 	balances := s.session.GetAccount().Balances()
 	for currency := range s.TargetWeights {
@@ -195,47 +197,37 @@ func (s *Strategy) balances() (types.BalanceMap, error) {
 	return m, nil
 }
 
-func (s *Strategy) generateSubmitOrders(ctx context.Context) (submitOrders []types.SubmitOrder, err error) {
-	prices, err := s.prices(ctx)
+func (s *Strategy) generateOrder(ctx context.Context) (*types.SubmitOrder, error) {
+	prices, err := s.queryMidPrices(ctx)
 	if err != nil {
 		return nil, err
 	}
-	balances, err := s.balances()
+
+	balances, err := s.selectBalances()
 	if err != nil {
 		return nil, err
 	}
-	marketValues := prices.Mul(balanceToTotal(balances))
-	currentWeights := marketValues.Normalize()
 
-	for currency, targetWeight := range s.TargetWeights {
-		if currency == s.QuoteCurrency {
-			continue
-		}
+	values := prices.Mul(toValueMap(balances))
+	weights := values.Normalize()
 
-		symbol := currency + s.QuoteCurrency
-		currentWeight := currentWeights[currency]
-		currentPrice := prices[currency]
+	for symbol, market := range s.markets {
+		target := s.TargetWeights[market.BaseCurrency]
+		weight := weights[market.BaseCurrency]
+		midPrice := prices[market.BaseCurrency]
 
-		log.Infof("%s price: %v, current weight: %v, target weight: %v",
-			symbol,
-			currentPrice,
-			currentWeight,
-			targetWeight)
+		log.Infof("%s mid price: %s", symbol, midPrice.String())
+		log.Infof("%s weight: %.2f%%, target: %.2f%%", market.BaseCurrency, weight.Float64()*100, target.Float64()*100)
 
 		// calculate the difference between current weight and target weight
 		// if the difference is less than threshold, then we will not create the order
-		weightDifference := targetWeight.Sub(currentWeight)
-		if weightDifference.Abs().Compare(s.Threshold) < 0 {
-			log.Infof("%s weight distance |%v - %v| = |%v| less than the threshold: %v",
-				symbol,
-				currentWeight,
-				targetWeight,
-				weightDifference,
-				s.Threshold)
+		diff := target.Sub(weight)
+		if diff.Abs().Compare(s.Threshold) < 0 {
+			log.Infof("%s weight is close to target, skip", market.BaseCurrency)
 			continue
 		}
 
-		quantity := weightDifference.Mul(marketValues.Sum()).Div(currentPrice)
+		quantity := diff.Mul(values.Sum()).Div(midPrice)
 
 		side := types.SideTypeBuy
 		if quantity.Sign() < 0 {
@@ -243,94 +235,47 @@ func (s *Strategy) generateSubmitOrders(ctx context.Context) (submitOrders []typ
 			quantity = quantity.Abs()
 		}
 
-		maxAmount := s.adjustMaxAmountByBalance(side, currency, currentPrice, balances)
-		if maxAmount.Sign() > 0 {
-			quantity = bbgo.AdjustQuantityByMaxAmount(quantity, currentPrice, maxAmount)
-			log.Infof("adjust the quantity %v (%s %s @ %v) by max amount %v",
-				quantity,
+		if s.MaxAmount.Float64() > 0 {
+			quantity = bbgo.AdjustQuantityByMaxAmount(quantity, midPrice, s.MaxAmount)
+			log.Infof("adjust quantity %s (%s %s @ %s) by max amount %s",
+				quantity.String(),
 				symbol,
 				side.String(),
-				currentPrice,
-				s.MaxAmount)
+				midPrice.String(),
+				s.MaxAmount.String())
 		}
 
-		log.Debugf("symbol: %v, quantity: %v", symbol, quantity)
+		if side == types.SideTypeBuy {
+			quantity = fixedpoint.Min(quantity, balances[s.QuoteCurrency].Available.Div(midPrice))
+		} else if side == types.SideTypeSell {
+			quantity = fixedpoint.Min(quantity, balances[market.BaseCurrency].Available)
+		}
 
-		order := types.SubmitOrder{
+		if market.IsDustQuantity(quantity, midPrice) {
+			log.Infof("quantity %s (%s %s @ %s) is dust quantity, skip",
+				quantity.String(),
+				symbol,
+				side.String(),
+				midPrice.String())
+			continue
+		}
+
+		return &types.SubmitOrder{
 			Symbol:   symbol,
 			Side:     side,
 			Type:     s.OrderType,
 			Quantity: quantity,
-			Price:    currentPrice,
-		}
-
-		if ok := s.checkMinimalOrderQuantity(order); ok {
-			submitOrders = append(submitOrders, order)
-		}
+			Price:    midPrice,
+		}, nil
 	}
-
-	return submitOrders, err
+	return nil, nil
 }
 
-func (s *Strategy) symbols() (symbols []string) {
-	for currency := range s.TargetWeights {
-		if currency == s.QuoteCurrency {
-			continue
-		}
-		symbols = append(symbols, currency+s.QuoteCurrency)
-	}
-	return symbols
-}
-
-func (s *Strategy) markets() ([]types.Market, error) {
-	markets := []types.Market{}
-	for _, symbol := range s.symbols() {
-		market, ok := s.session.Market(symbol)
-		if !ok {
-			return nil, fmt.Errorf("market %s not found", symbol)
-		}
-		markets = append(markets, market)
-	}
-	return markets, nil
-}
-
-func (s *Strategy) adjustMaxAmountByBalance(side types.SideType, currency string, currentPrice fixedpoint.Value, balances types.BalanceMap) fixedpoint.Value {
-	var maxAmount fixedpoint.Value
-
-	switch side {
-	case types.SideTypeBuy:
-		maxAmount = balances[s.QuoteCurrency].Available
-	case types.SideTypeSell:
-		maxAmount = balances[currency].Available.Mul(currentPrice)
-	default:
-		log.Errorf("unknown side type: %s", side)
-		return fixedpoint.Zero
-	}
-
-	if s.MaxAmount.Sign() > 0 {
-		maxAmount = fixedpoint.Min(s.MaxAmount, maxAmount)
-	}
-
-	return maxAmount
-}
-
-func (s *Strategy) checkMinimalOrderQuantity(order types.SubmitOrder) bool {
-	if order.Quantity.Compare(order.Market.MinQuantity) < 0 {
-		log.Infof("order quantity is too small: %f < %f", order.Quantity.Float64(), order.Market.MinQuantity.Float64())
-		return false
-	}
-
-	if order.Quantity.Mul(order.Price).Compare(order.Market.MinNotional) < 0 {
-		log.Infof("order min notional is too small: %f < %f", order.Quantity.Mul(order.Price).Float64(), order.Market.MinNotional.Float64())
-		return false
-	}
-	return true
-}
-
-func balanceToTotal(balances types.BalanceMap) types.ValueMap {
+func toValueMap(balances types.BalanceMap) types.ValueMap {
 	m := make(types.ValueMap)
 	for _, b := range balances {
-		m[b.Currency] = b.Total()
+		// m[b.Currency] = b.Net()
+		m[b.Currency] = b.Available
 	}
 	return m
 }

From 1d2e46eca8c9eebd89c5d4b8b596d05d17136a7b Mon Sep 17 00:00:00 2001
From: Yu-Cheng <yucheng@maicoin.com>
Date: Tue, 31 Oct 2023 12:36:59 +0800
Subject: [PATCH 34/42] trade: query trades from db paginately

---
 pkg/service/trade.go | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/pkg/service/trade.go b/pkg/service/trade.go
index ae8379fa0..9020290a7 100644
--- a/pkg/service/trade.go
+++ b/pkg/service/trade.go
@@ -23,7 +23,12 @@ type QueryTradesOptions struct {
 	Sessions []string
 	Symbol   string
 	LastGID  int64
-	Since    *time.Time
+
+	// inclusive
+	Since *time.Time
+
+	// exclusive
+	Until *time.Time
 
 	// ASC or DESC
 	Ordering string
@@ -272,11 +277,19 @@ func (s *TradeService) Query(options QueryTradesOptions) ([]types.Trade, error)
 	sel := sq.Select("*").
 		From("trades")
 
+	if options.LastGID != 0 {
+		sel = sel.Where(sq.Gt{"gid": options.LastGID})
+	}
 	if options.Since != nil {
 		sel = sel.Where(sq.GtOrEq{"traded_at": options.Since})
 	}
+	if options.Until != nil {
+		sel = sel.Where(sq.Lt{"traded_at": options.Until})
+	}
 
-	sel = sel.Where(sq.Eq{"symbol": options.Symbol})
+	if options.Symbol != "" {
+		sel = sel.Where(sq.Eq{"symbol": options.Symbol})
+	}
 
 	if options.Exchange != "" {
 		sel = sel.Where(sq.Eq{"exchange": options.Exchange})
@@ -412,4 +425,3 @@ func SelectLastTrades(ex types.ExchangeName, symbol string, isMargin, isFutures,
 		OrderBy("traded_at DESC").
 		Limit(limit)
 }
-

From 4bc177f21bd0be289fd3d75753683447e26c1f55 Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Tue, 31 Oct 2023 11:56:15 +0800
Subject: [PATCH 35/42] pkg/exchange: refactor get symbol api

---
 .../bitget/bitgetapi/get_symbols_request.go   | 17 ++++-
 pkg/exchange/bitget/convert.go                | 23 +++++++
 pkg/exchange/bitget/convert_test.go           | 66 +++++++++++++++++++
 pkg/exchange/bitget/exchange.go               | 32 ++++-----
 4 files changed, 116 insertions(+), 22 deletions(-)
 create mode 100644 pkg/exchange/bitget/convert_test.go

diff --git a/pkg/exchange/bitget/bitgetapi/get_symbols_request.go b/pkg/exchange/bitget/bitgetapi/get_symbols_request.go
index c3c4e64a9..48a202a2e 100644
--- a/pkg/exchange/bitget/bitgetapi/get_symbols_request.go
+++ b/pkg/exchange/bitget/bitgetapi/get_symbols_request.go
@@ -9,6 +9,17 @@ import (
 	"github.com/c9s/bbgo/pkg/fixedpoint"
 )
 
+type SymbolStatus string
+
+const (
+	// SymbolOffline represent market is suspended, users cannot trade.
+	SymbolOffline SymbolStatus = "offline"
+	// SymbolGray represents market is online, but user trading is not available.
+	SymbolGray SymbolStatus = "gray"
+	// SymbolOnline trading begins, users can trade.
+	SymbolOnline SymbolStatus = "online"
+)
+
 type Symbol struct {
 	Symbol              string           `json:"symbol"`
 	SymbolName          string           `json:"symbolName"`
@@ -18,10 +29,10 @@ type Symbol struct {
 	MaxTradeAmount      fixedpoint.Value `json:"maxTradeAmount"`
 	TakerFeeRate        fixedpoint.Value `json:"takerFeeRate"`
 	MakerFeeRate        fixedpoint.Value `json:"makerFeeRate"`
-	PriceScale          int              `json:"priceScale"`
-	QuantityScale       int              `json:"quantityScale"`
+	PriceScale          fixedpoint.Value `json:"priceScale"`
+	QuantityScale       fixedpoint.Value `json:"quantityScale"`
 	MinTradeUSDT        fixedpoint.Value `json:"minTradeUSDT"`
-	Status              string           `json:"status"`
+	Status              SymbolStatus     `json:"status"`
 	BuyLimitPriceRatio  fixedpoint.Value `json:"buyLimitPriceRatio"`
 	SellLimitPriceRatio fixedpoint.Value `json:"sellLimitPriceRatio"`
 }
diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go
index 0836b4dd0..f40d1d58a 100644
--- a/pkg/exchange/bitget/convert.go
+++ b/pkg/exchange/bitget/convert.go
@@ -1,6 +1,7 @@
 package bitget
 
 import (
+	"math"
 	"strings"
 
 	"github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi"
@@ -23,3 +24,25 @@ func toGlobalBalance(asset bitgetapi.AccountAsset) types.Balance {
 		MaxWithdrawAmount: fixedpoint.Zero,
 	}
 }
+
+func toGlobalMarket(s bitgetapi.Symbol) types.Market {
+	if s.Status != bitgetapi.SymbolOnline {
+		log.Warnf("The symbol %s is not online", s.Symbol)
+	}
+	return types.Market{
+		Symbol:          s.SymbolName,
+		LocalSymbol:     s.Symbol,
+		PricePrecision:  s.PriceScale.Int(),
+		VolumePrecision: s.QuantityScale.Int(),
+		QuoteCurrency:   s.QuoteCoin,
+		BaseCurrency:    s.BaseCoin,
+		MinNotional:     s.MinTradeUSDT,
+		MinAmount:       s.MinTradeUSDT,
+		MinQuantity:     s.MinTradeAmount,
+		MaxQuantity:     s.MaxTradeAmount,
+		StepSize:        fixedpoint.NewFromFloat(1.0 / math.Pow10(s.QuantityScale.Int())),
+		TickSize:        fixedpoint.NewFromFloat(1.0 / math.Pow10(s.PriceScale.Int())),
+		MinPrice:        fixedpoint.Zero,
+		MaxPrice:        fixedpoint.Zero,
+	}
+}
diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go
new file mode 100644
index 000000000..6ab315eaa
--- /dev/null
+++ b/pkg/exchange/bitget/convert_test.go
@@ -0,0 +1,66 @@
+package bitget
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi"
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+func TestToGlobalMarket(t *testing.T) {
+	// sample:
+	//{
+	//            "symbol":"BTCUSDT_SPBL",
+	//            "symbolName":"BTCUSDT",
+	//            "baseCoin":"BTC",
+	//            "quoteCoin":"USDT",
+	//            "minTradeAmount":"0.0001",
+	//            "maxTradeAmount":"10000",
+	//            "takerFeeRate":"0.001",
+	//            "makerFeeRate":"0.001",
+	//            "priceScale":"4",
+	//            "quantityScale":"8",
+	//            "minTradeUSDT":"5",
+	//            "status":"online",
+	//            "buyLimitPriceRatio": "0.05",
+	//            "sellLimitPriceRatio": "0.05"
+	//        }
+	inst := bitgetapi.Symbol{
+		Symbol:              "BTCUSDT_SPBL",
+		SymbolName:          "BTCUSDT",
+		BaseCoin:            "BTC",
+		QuoteCoin:           "USDT",
+		MinTradeAmount:      fixedpoint.NewFromFloat(0.0001),
+		MaxTradeAmount:      fixedpoint.NewFromFloat(10000),
+		TakerFeeRate:        fixedpoint.NewFromFloat(0.001),
+		MakerFeeRate:        fixedpoint.NewFromFloat(0.001),
+		PriceScale:          fixedpoint.NewFromFloat(4),
+		QuantityScale:       fixedpoint.NewFromFloat(8),
+		MinTradeUSDT:        fixedpoint.NewFromFloat(5),
+		Status:              bitgetapi.SymbolOnline,
+		BuyLimitPriceRatio:  fixedpoint.NewFromFloat(0.05),
+		SellLimitPriceRatio: fixedpoint.NewFromFloat(0.05),
+	}
+
+	exp := types.Market{
+		Symbol:          inst.SymbolName,
+		LocalSymbol:     inst.Symbol,
+		PricePrecision:  4,
+		VolumePrecision: 8,
+		QuoteCurrency:   inst.QuoteCoin,
+		BaseCurrency:    inst.BaseCoin,
+		MinNotional:     inst.MinTradeUSDT,
+		MinAmount:       inst.MinTradeUSDT,
+		MinQuantity:     inst.MinTradeAmount,
+		MaxQuantity:     inst.MaxTradeAmount,
+		StepSize:        fixedpoint.NewFromFloat(0.00000001),
+		MinPrice:        fixedpoint.Zero,
+		MaxPrice:        fixedpoint.Zero,
+		TickSize:        fixedpoint.NewFromFloat(0.0001),
+	}
+
+	assert.Equal(t, toGlobalMarket(inst), exp)
+}
diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go
index 0d17c1b00..b7af3828b 100644
--- a/pkg/exchange/bitget/exchange.go
+++ b/pkg/exchange/bitget/exchange.go
@@ -2,12 +2,13 @@ package bitget
 
 import (
 	"context"
-	"math"
+	"fmt"
+	"time"
 
 	"github.com/sirupsen/logrus"
+	"golang.org/x/time/rate"
 
 	"github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi"
-	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/types"
 )
 
@@ -19,6 +20,11 @@ var log = logrus.WithFields(logrus.Fields{
 	"exchange": ID,
 })
 
+var (
+	// queryMarketRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-symbols
+	queryMarketRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5)
+)
+
 type Exchange struct {
 	key, secret, passphrase string
 
@@ -54,7 +60,10 @@ func (e *Exchange) NewStream() types.Stream {
 }
 
 func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) {
-	// TODO implement me
+	if err := queryMarketRateLimiter.Wait(ctx); err != nil {
+		return nil, fmt.Errorf("markets rate limiter wait error: %w", err)
+	}
+
 	req := e.client.NewGetSymbolsRequest()
 	symbols, err := req.Do(ctx)
 	if err != nil {
@@ -64,22 +73,7 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) {
 	markets := types.MarketMap{}
 	for _, s := range symbols {
 		symbol := toGlobalSymbol(s.SymbolName)
-		markets[symbol] = types.Market{
-			Symbol:          s.SymbolName,
-			LocalSymbol:     s.Symbol,
-			PricePrecision:  s.PriceScale,
-			VolumePrecision: s.QuantityScale,
-			QuoteCurrency:   s.QuoteCoin,
-			BaseCurrency:    s.BaseCoin,
-			MinNotional:     s.MinTradeUSDT,
-			MinAmount:       s.MinTradeUSDT,
-			MinQuantity:     s.MinTradeAmount,
-			MaxQuantity:     s.MaxTradeAmount,
-			StepSize:        fixedpoint.NewFromFloat(math.Pow10(-s.QuantityScale)),
-			TickSize:        fixedpoint.NewFromFloat(math.Pow10(-s.PriceScale)),
-			MinPrice:        fixedpoint.Zero,
-			MaxPrice:        fixedpoint.Zero,
-		}
+		markets[symbol] = toGlobalMarket(s)
 	}
 
 	return markets, nil

From 64cad567274d8c0637dd5e50eef779490ff32478 Mon Sep 17 00:00:00 2001
From: Himanshu Kumar Mahto <93067059+HimanshuMahto@users.noreply.github.com>
Date: Wed, 1 Nov 2023 02:20:58 +0530
Subject: [PATCH 36/42] grammatical errors in the README.md

---
 README.md | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index 12af0d06f..762f0b082 100644
--- a/README.md
+++ b/README.md
@@ -101,25 +101,25 @@ the implementation.
 | xnav        | this strategy helps you record the current net asset value                                                                              | tool       | no               |
 | xalign      | this strategy aligns your balance position automatically                                                                                | tool       | no               |
 | xfunding    | a funding rate fee strategy                                                                                                             | funding    | no               |
-| autoborrow  | this strategy uses margin to borrow assets, to help you keep the minimal balance                                                        | tool       | no               |
-| pivotshort  | this strategy finds the pivot low and entry the trade when the price breaks the previous low                                            | long/short |                  |
+| autoborrow  | this strategy uses margin to borrow assets, to help you keep a minimal balance                                                        | tool       | no               |
+| pivotshort  | this strategy finds the pivot low and enters the trade when the price breaks the previous low                                            | long/short |                  |
 | schedule    | this strategy buy/sell with a fixed quantity periodically, you can use this as a single DCA, or to refill the fee asset like BNB.       | tool       |
 | irr         | this strategy opens the position based on the predicated return rate                                                                    | long/short |                  |
-| bollmaker   | this strategy holds a long-term long/short position, places maker orders on both side, uses bollinger band to control the position size | maker      |                  |
-| wall        | this strategy creates wall (large amount order) on the order book                                                                       | maker      | no               |
+| bollmaker   | this strategy holds a long-term long/short position, places maker orders on both sides, and uses a bollinger band to control the position size | maker      |                  |
+| wall        | this strategy creates a wall (large amount of order) on the order book                                                                       | maker      | no               |
 | scmaker     | this market making strategy is designed for stable coin markets, like USDC/USDT                                                         | maker      |                  |
 | drift       |                                                                                                                                         | long/short |                  |
-| rsicross    | this strategy opens a long position when the fast rsi cross over the slow rsi, this is a demo strategy for using the v2 indicator       | long/short |                  |
+| rsicross    | this strategy opens a long position when the fast rsi crosses over the slow rsi, this is a demo strategy for using the v2 indicator       | long/short |                  |
 | marketcap   | this strategy implements a strategy that rebalances the portfolio based on the market capitalization                                    | rebalance  | no               |
 | supertrend  | this strategy uses DEMA and Supertrend indicator to open the long/short position                                                        | long/short |                  |
-| trendtrader | this strategy opens long/short position based on the trendline breakout                                                                 | long/short |                  |
+| trendtrader | this strategy opens a long/short position based on the trendline breakout                                                                 | long/short |                  |
 | elliottwave |                                                                                                                                         | long/short |                  |
 | ewoDgtrd    |                                                                                                                                         | long/short |                  |
 | fixedmaker  |                                                                                                                                         | maker      |                  |
 | factoryzoo  |                                                                                                                                         | long/short |                  |
 | fmaker      |                                                                                                                                         | maker      |                  |
 | linregmaker | a linear regression based market maker                                                                                                  | maker      |                  |
-| convert     | convert strategy is a tool that helps you convert specific asset to a target asset                                                      | tool       | no               |
+| convert     | convert strategy is a tool that helps you convert a specific asset to a target asset                                                      | tool       | no               |
 
 
 
@@ -250,7 +250,7 @@ To start bbgo with the frontend dashboard:
 bbgo run --enable-webserver
 ```
 
-If you want to switch to other dotenv file, you can add an `--dotenv` option or `--config`:
+If you want to switch to another dotenv file, you can add an `--dotenv` option or `--config`:
 
 ```sh
 bbgo sync --dotenv .env.dev --config config/grid.yaml --session binance
@@ -292,7 +292,7 @@ You could also add the script to crontab so that the system time could get synch
 
 ### Testnet (Paper Trading)
 
-Currently only supports binance testnet. To run bbgo in testnet, apply new API keys
+Currently only supports Binance testnet. To run bbgo in testnet, apply new API keys
 from [Binance Test Network](https://testnet.binance.vision), and set the following env before you start bbgo:
 
 ```bash
@@ -319,7 +319,7 @@ You can only use one database driver MySQL or SQLite to store your trading data.
 
 #### Configure MySQL Database
 
-To use MySQL database for data syncing, first, you need to install your mysql server:
+To use MySQL database for data syncing, first, you need to install your MySQL server:
 
 ```sh
 # For Ubuntu Linux
@@ -406,7 +406,7 @@ Check out the strategy directory [strategy](pkg/strategy) for all built-in strat
 - `drift` - drift strategy.
 - `grid2` - the second-generation grid strategy.
 
-To run these built-in strategies, just modify the config file to make the configuration suitable for you, for example if
+To run these built-in strategies, just modify the config file to make the configuration suitable for you, for example, if
 you want to run
 `buyandhold` strategy:
 
@@ -524,7 +524,7 @@ bbgo userdatastream --session binance
 
 In order to minimize the strategy code, bbgo supports dynamic dependency injection.
 
-Before executing your strategy, bbgo injects the components into your strategy object if it found the embedded field
+Before executing your strategy, bbgo injects the components into your strategy object if it finds the embedded field
 that is using bbgo component. for example:
 
 ```go
@@ -591,7 +591,7 @@ streambook.BindStream(stream)
 
 1. Click the "Fork" button from the GitHub repository.
 2. Clone your forked repository into `$GOPATH/github.com/c9s/bbgo`.
-3. Change the directory into `$GOPATH/github.com/c9s/bbgo`.
+3. Change the directory to `$GOPATH/github.com/c9s/bbgo`.
 4. Create a branch and start your development.
 5. Test your changes.
 6. Push your changes to your fork.
@@ -621,7 +621,7 @@ make embed && go run -tags web ./cmd/bbgo-lorca
 ### Looking For A New Strategy?
 
 You can write an article about BBGO on any topic, in 750-1500 words for exchange, and I can implement the strategy for
-you (depending on the complexity and efforts). If you're interested in, DM me in telegram <https://t.me/c123456789s> or
+you (depending on the complexity and effort). If you're interested in, DM me in telegram <https://t.me/c123456789s> or
 twitter <https://twitter.com/c9s>, and we can discuss.
 
 ### Adding New Crypto Exchange support?

From 102b662f7c6ebb763a1bc5f44428f9550f87735c Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Tue, 31 Oct 2023 21:53:13 +0800
Subject: [PATCH 37/42] pkg/exchange: support kline subscription on stream

---
 pkg/exchange/bitget/stream.go           |  76 +++++++--
 pkg/exchange/bitget/stream_callbacks.go |  10 ++
 pkg/exchange/bitget/stream_test.go      | 201 ++++++++++++++++++++++++
 pkg/exchange/bitget/types.go            | 132 ++++++++++++++++
 pkg/exchange/bitget/types_test.go       |  43 +++++
 5 files changed, 451 insertions(+), 11 deletions(-)
 create mode 100644 pkg/exchange/bitget/types_test.go

diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go
index 5a2f6f23d..039c65127 100644
--- a/pkg/exchange/bitget/stream.go
+++ b/pkg/exchange/bitget/stream.go
@@ -5,17 +5,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"time"
+	"github.com/gorilla/websocket"
+	"strings"
 
 	"github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi"
 	"github.com/c9s/bbgo/pkg/types"
-	"github.com/gorilla/websocket"
-)
-
-const (
-	// Client should keep ping the server in every 30 seconds. Server will close the connections which has no ping over
-	// 120 seconds(even when the client is still receiving data from the server)
-	pingInterval = 30 * time.Second
 )
 
 var (
@@ -29,11 +23,15 @@ type Stream struct {
 
 	bookEventCallbacks        []func(o BookEvent)
 	marketTradeEventCallbacks []func(o MarketTradeEvent)
+	KLineEventCallbacks       []func(o KLineEvent)
+
+	lastCandle map[string]types.KLine
 }
 
 func NewStream() *Stream {
 	stream := &Stream{
 		StandardStream: types.NewStandardStream(),
+		lastCandle:     map[string]types.KLine{},
 	}
 
 	stream.SetEndpointCreator(stream.createEndpoint)
@@ -44,6 +42,7 @@ func NewStream() *Stream {
 
 	stream.OnBookEvent(stream.handleBookEvent)
 	stream.OnMarketTradeEvent(stream.handleMaretTradeEvent)
+	stream.OnKLineEvent(stream.handleKLineEvent)
 	return stream
 }
 
@@ -108,6 +107,9 @@ func (s *Stream) dispatchEvent(event interface{}) {
 	case *MarketTradeEvent:
 		s.EmitMarketTradeEvent(*e)
 
+	case *KLineEvent:
+		s.EmitKLineEvent(*e)
+
 	case []byte:
 		// We only handle the 'pong' case. Others are unexpected.
 		if !bytes.Equal(e, pongBytes) {
@@ -171,6 +173,15 @@ func convertSubscription(sub types.Subscription) (WsArg, error) {
 	case types.MarketTradeChannel:
 		arg.Channel = ChannelTrade
 		return arg, nil
+
+	case types.KLineChannel:
+		interval, found := toLocalInterval[sub.Options.Interval]
+		if !found {
+			return WsArg{}, fmt.Errorf("interval %s not supported on KLine subscription", sub.Options.Interval)
+		}
+
+		arg.Channel = ChannelType(interval)
+		return arg, nil
 	}
 
 	return arg, fmt.Errorf("unsupported stream channel: %s", sub.Channel)
@@ -200,7 +211,8 @@ func parseEvent(in []byte) (interface{}, error) {
 		return &event, nil
 	}
 
-	switch event.Arg.Channel {
+	ch := event.Arg.Channel
+	switch ch {
 	case ChannelOrderBook, ChannelOrderBook5, ChannelOrderBook15:
 		var book BookEvent
 		err = json.Unmarshal(event.Data, &book.Events)
@@ -222,9 +234,26 @@ func parseEvent(in []byte) (interface{}, error) {
 		trade.actionType = event.Action
 		trade.instId = event.Arg.InstId
 		return &trade, nil
-	}
 
-	return nil, fmt.Errorf("unhandled websocket event: %+v", string(in))
+	default:
+
+		// handle the `KLine` case here to avoid complicating the code structure.
+		if strings.HasPrefix(string(ch), "candle") {
+			var kline KLineEvent
+			err = json.Unmarshal(event.Data, &kline.Events)
+			if err != nil {
+				return nil, fmt.Errorf("failed to unmarshal data into KLineEvent, Arg: %+v Data: %s, err: %w", event.Arg, string(event.Data), err)
+			}
+
+			kline.actionType = event.Action
+			kline.channel = ch
+			kline.instId = event.Arg.InstId
+			return &kline, nil
+		}
+		// return an error for any other case
+
+		return nil, fmt.Errorf("unhandled websocket event: %+v", string(in))
+	}
 }
 
 func (s *Stream) handleMaretTradeEvent(m MarketTradeEvent) {
@@ -242,3 +271,28 @@ func (s *Stream) handleMaretTradeEvent(m MarketTradeEvent) {
 		s.EmitMarketTrade(globalTrade)
 	}
 }
+
+func (s *Stream) handleKLineEvent(k KLineEvent) {
+	if k.actionType == ActionTypeSnapshot {
+		// we don't support snapshot event
+		return
+	}
+
+	interval, found := toGlobalInterval[string(k.channel)]
+	if !found {
+		log.Errorf("unexpected interval %s on KLine subscription", k.channel)
+		return
+	}
+
+	for _, kline := range k.Events {
+		last, ok := s.lastCandle[k.CacheKey()]
+		if ok && kline.StartTime.Time().After(last.StartTime.Time()) {
+			last.Closed = true
+			s.EmitKLineClosed(last)
+		}
+
+		kLine := kline.ToGlobal(interval, k.instId)
+		s.EmitKLine(kLine)
+		s.lastCandle[k.CacheKey()] = kLine
+	}
+}
diff --git a/pkg/exchange/bitget/stream_callbacks.go b/pkg/exchange/bitget/stream_callbacks.go
index 01da4388f..82ef7beae 100644
--- a/pkg/exchange/bitget/stream_callbacks.go
+++ b/pkg/exchange/bitget/stream_callbacks.go
@@ -23,3 +23,13 @@ func (s *Stream) EmitMarketTradeEvent(o MarketTradeEvent) {
 		cb(o)
 	}
 }
+
+func (s *Stream) OnKLineEvent(cb func(o KLineEvent)) {
+	s.KLineEventCallbacks = append(s.KLineEventCallbacks, cb)
+}
+
+func (s *Stream) EmitKLineEvent(o KLineEvent) {
+	for _, cb := range s.KLineEventCallbacks {
+		cb(o)
+	}
+}
diff --git a/pkg/exchange/bitget/stream_test.go b/pkg/exchange/bitget/stream_test.go
index c25fcd942..b33e6afa2 100644
--- a/pkg/exchange/bitget/stream_test.go
+++ b/pkg/exchange/bitget/stream_test.go
@@ -106,6 +106,22 @@ func TestStream(t *testing.T) {
 		<-c
 	})
 
+	t.Run("kline test", func(t *testing.T) {
+		s.Subscribe(types.KLineChannel, "BTCUSDT", types.SubscribeOptions{Interval: types.Interval1w})
+		s.SetPublicOnly()
+		err := s.Connect(context.Background())
+		assert.NoError(t, err)
+
+		s.OnKLine(func(kline types.KLine) {
+			t.Log("got update", kline)
+		})
+		s.OnKLineClosed(func(kline types.KLine) {
+			t.Log("got closed update", kline)
+		})
+		c := make(chan struct{})
+		<-c
+	})
+
 }
 
 func TestStream_parseWebSocketEvent(t *testing.T) {
@@ -453,6 +469,174 @@ func Test_parseWebSocketEvent_MarketTrade(t *testing.T) {
 	})
 }
 
+func Test_parseWebSocketEvent_KLine(t *testing.T) {
+	t.Run("KLine event", func(t *testing.T) {
+		input := `{
+		   "action":"%s",
+		   "arg":{
+			  "instType":"sp",
+			  "channel":"candle5m",
+			  "instId":"BTCUSDT"
+		   },
+		   "data":[
+				["1698744600000","34361.49","34458.98","34355.53","34416.41","99.6631"]
+		   ],
+		   "ts":1697697791670
+		}`
+
+		eventFn := func(in string, actionType ActionType) {
+			res, err := parseWebSocketEvent([]byte(in))
+			assert.NoError(t, err)
+			kline, ok := res.(*KLineEvent)
+			assert.True(t, ok)
+			assert.Equal(t, KLineEvent{
+				channel: "candle5m",
+				Events: KLineSlice{
+					{
+						StartTime:    types.NewMillisecondTimestampFromInt(1698744600000),
+						OpenPrice:    fixedpoint.NewFromFloat(34361.49),
+						HighestPrice: fixedpoint.NewFromFloat(34458.98),
+						LowestPrice:  fixedpoint.NewFromFloat(34355.53),
+						ClosePrice:   fixedpoint.NewFromFloat(34416.41),
+						Volume:       fixedpoint.NewFromFloat(99.6631),
+					},
+				},
+				actionType: actionType,
+				instId:     "BTCUSDT",
+			}, *kline)
+		}
+
+		t.Run("snapshot type", func(t *testing.T) {
+			snapshotInput := fmt.Sprintf(input, ActionTypeSnapshot)
+			eventFn(snapshotInput, ActionTypeSnapshot)
+		})
+
+		t.Run("update type", func(t *testing.T) {
+			snapshotInput := fmt.Sprintf(input, ActionTypeUpdate)
+			eventFn(snapshotInput, ActionTypeUpdate)
+		})
+	})
+
+	t.Run("Unexpected length of kline", func(t *testing.T) {
+		input := `{
+		   "action":"%s",
+		   "arg":{
+			  "instType":"sp",
+			  "channel":"candle5m",
+			  "instId":"BTCUSDT"
+		   },
+		   "data":[
+			  ["1698744600000","34361.45","34458.98","34355.53","34416.41","99.6631", "123456"]
+		   ],
+		   "ts":1697697791670
+		}`
+		_, err := parseWebSocketEvent([]byte(input))
+		assert.ErrorContains(t, err, "unexpected kline length")
+	})
+
+	t.Run("Unexpected timestamp", func(t *testing.T) {
+		input := `{
+		   "action":"%s",
+		   "arg":{
+			  "instType":"sp",
+			  "channel":"candle5m",
+			  "instId":"BTCUSDT"
+		   },
+		   "data":[
+			  ["timestamp","34361.49","34458.98","34355.53","34416.41","99.6631"]
+		   ],
+		   "ts":1697697791670
+		}`
+		_, err := parseWebSocketEvent([]byte(input))
+		assert.ErrorContains(t, err, "timestamp")
+	})
+
+	t.Run("Unexpected open price", func(t *testing.T) {
+		input := `{
+		   "action":"%s",
+		   "arg":{
+			  "instType":"sp",
+			  "channel":"candle5m",
+			  "instId":"BTCUSDT"
+		   },
+		   "data":[
+				["1698744600000","1p","34458.98","34355.53","34416.41","99.6631"]
+		   ],
+		   "ts":1697697791670
+		}`
+		_, err := parseWebSocketEvent([]byte(input))
+		assert.ErrorContains(t, err, "open price")
+	})
+
+	t.Run("Unexpected highest price", func(t *testing.T) {
+		input := `{
+		   "action":"%s",
+		   "arg":{
+			  "instType":"sp",
+			  "channel":"candle5m",
+			  "instId":"BTCUSDT"
+		   },
+		   "data":[
+			  ["1698744600000","34361.45","3p","34355.53","34416.41","99.6631"]
+		   ],
+		   "ts":1697697791670
+		}`
+		_, err := parseWebSocketEvent([]byte(input))
+		assert.ErrorContains(t, err, "highest price")
+	})
+
+	t.Run("Unexpected lowest price", func(t *testing.T) {
+		input := `{
+		   "action":"%s",
+		   "arg":{
+			  "instType":"sp",
+			  "channel":"candle5m",
+			  "instId":"BTCUSDT"
+		   },
+		   "data":[
+			  ["1698744600000","34361.45","34458.98","1p","34416.41","99.6631"]
+		   ],
+		   "ts":1697697791670
+		}`
+		_, err := parseWebSocketEvent([]byte(input))
+		assert.ErrorContains(t, err, "lowest price")
+	})
+
+	t.Run("Unexpected close price", func(t *testing.T) {
+		input := `{
+		   "action":"%s",
+		   "arg":{
+			  "instType":"sp",
+			  "channel":"candle5m",
+			  "instId":"BTCUSDT"
+		   },
+		   "data":[
+			  ["1698744600000","34361.45","34458.98","34355.53","1c","99.6631"]
+		   ],
+		   "ts":1697697791670
+		}`
+		_, err := parseWebSocketEvent([]byte(input))
+		assert.ErrorContains(t, err, "close price")
+	})
+
+	t.Run("Unexpected volume", func(t *testing.T) {
+		input := `{
+		   "action":"%s",
+		   "arg":{
+			  "instType":"sp",
+			  "channel":"candle5m",
+			  "instId":"BTCUSDT"
+		   },
+		   "data":[
+			  ["1698744600000","34361.45","34458.98","34355.53","34416.41", "1v"]
+		   ],
+		   "ts":1697697791670
+		}`
+		_, err := parseWebSocketEvent([]byte(input))
+		assert.ErrorContains(t, err, "volume")
+	})
+}
+
 func Test_convertSubscription(t *testing.T) {
 	t.Run("BookChannel.ChannelOrderBook5", func(t *testing.T) {
 		res, err := convertSubscription(types.Subscription{
@@ -512,4 +696,21 @@ func Test_convertSubscription(t *testing.T) {
 			InstId:   "BTCUSDT",
 		}, res)
 	})
+	t.Run("CandleChannel", func(t *testing.T) {
+		for gInterval, localInterval := range toLocalInterval {
+			res, err := convertSubscription(types.Subscription{
+				Symbol:  "BTCUSDT",
+				Channel: types.KLineChannel,
+				Options: types.SubscribeOptions{
+					Interval: gInterval,
+				},
+			})
+			assert.NoError(t, err)
+			assert.Equal(t, WsArg{
+				InstType: instSp,
+				Channel:  ChannelType(localInterval),
+				InstId:   "BTCUSDT",
+			}, res)
+		}
+	})
 }
diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go
index df75f5bf2..a1107cad6 100644
--- a/pkg/exchange/bitget/types.go
+++ b/pkg/exchange/bitget/types.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/types"
@@ -260,3 +261,134 @@ type MarketTradeEvent struct {
 	actionType ActionType
 	instId     string
 }
+
+var (
+	toLocalInterval = map[types.Interval]string{
+		types.Interval1m:  "candle1m",
+		types.Interval5m:  "candle5m",
+		types.Interval15m: "candle15m",
+		types.Interval30m: "candle30m",
+		types.Interval1h:  "candle1H",
+		types.Interval4h:  "candle4H",
+		types.Interval12h: "candle12H",
+		types.Interval1d:  "candle1D",
+		types.Interval1w:  "candle1W",
+	}
+
+	toGlobalInterval = map[string]types.Interval{
+		"candle1m":  types.Interval1m,
+		"candle5m":  types.Interval5m,
+		"candle15m": types.Interval15m,
+		"candle30m": types.Interval30m,
+		"candle1H":  types.Interval1h,
+		"candle4H":  types.Interval4h,
+		"candle12H": types.Interval12h,
+		"candle1D":  types.Interval1d,
+		"candle1W":  types.Interval1w,
+	}
+)
+
+type KLine struct {
+	StartTime    types.MillisecondTimestamp
+	OpenPrice    fixedpoint.Value
+	HighestPrice fixedpoint.Value
+	LowestPrice  fixedpoint.Value
+	ClosePrice   fixedpoint.Value
+	Volume       fixedpoint.Value
+}
+
+func (k KLine) ToGlobal(interval types.Interval, symbol string) types.KLine {
+	startTime := k.StartTime.Time()
+
+	return types.KLine{
+		Exchange:                 types.ExchangeBitget,
+		Symbol:                   symbol,
+		StartTime:                types.Time(startTime),
+		EndTime:                  types.Time(startTime.Add(interval.Duration() - time.Millisecond)),
+		Interval:                 interval,
+		Open:                     k.OpenPrice,
+		Close:                    k.ClosePrice,
+		High:                     k.HighestPrice,
+		Low:                      k.LowestPrice,
+		Volume:                   k.Volume,
+		QuoteVolume:              fixedpoint.Zero, // not supported
+		TakerBuyBaseAssetVolume:  fixedpoint.Zero, // not supported
+		TakerBuyQuoteAssetVolume: fixedpoint.Zero, // not supported
+		LastTradeID:              0,               // not supported
+		NumberOfTrades:           0,               // not supported
+		Closed:                   false,
+	}
+}
+
+type KLineSlice []KLine
+
+func (m *KLineSlice) UnmarshalJSON(b []byte) error {
+	if m == nil {
+		return errors.New("nil pointer of kline slice")
+	}
+	s, err := parseKLineSliceJSON(b)
+	if err != nil {
+		return err
+	}
+
+	*m = s
+	return nil
+}
+
+// parseKLineSliceJSON tries to parse a 2 dimensional string array into a KLineSlice
+//
+//	[
+//
+//	    ["1597026383085", "8533.02", "8553.74", "8527.17", "8548.26", "45247"]
+//	]
+func parseKLineSliceJSON(in []byte) (slice KLineSlice, err error) {
+	var rawKLines [][]json.RawMessage
+
+	err = json.Unmarshal(in, &rawKLines)
+	if err != nil {
+		return slice, err
+	}
+
+	for _, raw := range rawKLines {
+		if len(raw) != 6 {
+			return nil, fmt.Errorf("unexpected kline length: %d, data: %q", len(raw), raw)
+		}
+		var kline KLine
+		if err = json.Unmarshal(raw[0], &kline.StartTime); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal into timestamp: %q", raw[0])
+		}
+		if err = json.Unmarshal(raw[1], &kline.OpenPrice); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal into open price: %q", raw[1])
+		}
+		if err = json.Unmarshal(raw[2], &kline.HighestPrice); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal into highest price: %q", raw[2])
+		}
+		if err = json.Unmarshal(raw[3], &kline.LowestPrice); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal into lowest price: %q", raw[3])
+		}
+		if err = json.Unmarshal(raw[4], &kline.ClosePrice); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal into close price: %q", raw[4])
+		}
+		if err = json.Unmarshal(raw[5], &kline.Volume); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal into volume: %q", raw[5])
+		}
+
+		slice = append(slice, kline)
+	}
+
+	return slice, nil
+}
+
+type KLineEvent struct {
+	Events KLineSlice
+
+	// internal use
+	actionType ActionType
+	channel    ChannelType
+	instId     string
+}
+
+func (k KLineEvent) CacheKey() string {
+	// e.q: candle5m.BTCUSDT
+	return fmt.Sprintf("%s.%s", k.channel, k.instId)
+}
diff --git a/pkg/exchange/bitget/types_test.go b/pkg/exchange/bitget/types_test.go
new file mode 100644
index 000000000..185056620
--- /dev/null
+++ b/pkg/exchange/bitget/types_test.go
@@ -0,0 +1,43 @@
+package bitget
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+func TestKLine_ToGlobal(t *testing.T) {
+	startTime := int64(1698744600000)
+	interval := types.Interval1m
+	k := KLine{
+		StartTime:    types.NewMillisecondTimestampFromInt(startTime),
+		OpenPrice:    fixedpoint.NewFromFloat(34361.49),
+		HighestPrice: fixedpoint.NewFromFloat(34458.98),
+		LowestPrice:  fixedpoint.NewFromFloat(34355.53),
+		ClosePrice:   fixedpoint.NewFromFloat(34416.41),
+		Volume:       fixedpoint.NewFromFloat(99.6631),
+	}
+
+	assert.Equal(t, types.KLine{
+		Exchange:                 types.ExchangeBitget,
+		Symbol:                   "BTCUSDT",
+		StartTime:                types.Time(types.NewMillisecondTimestampFromInt(startTime).Time()),
+		EndTime:                  types.Time(types.NewMillisecondTimestampFromInt(startTime).Time().Add(interval.Duration() - time.Millisecond)),
+		Interval:                 interval,
+		Open:                     fixedpoint.NewFromFloat(34361.49),
+		Close:                    fixedpoint.NewFromFloat(34416.41),
+		High:                     fixedpoint.NewFromFloat(34458.98),
+		Low:                      fixedpoint.NewFromFloat(34355.53),
+		Volume:                   fixedpoint.NewFromFloat(99.6631),
+		QuoteVolume:              fixedpoint.Zero,
+		TakerBuyBaseAssetVolume:  fixedpoint.Zero,
+		TakerBuyQuoteAssetVolume: fixedpoint.Zero,
+		LastTradeID:              0,
+		NumberOfTrades:           0,
+		Closed:                   false,
+	}, k.ToGlobal(interval, "BTCUSDT"))
+}

From 2cea0894043d2b00f3d898a26ac0c0b8eb3f5aee Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Wed, 1 Nov 2023 11:46:43 +0800
Subject: [PATCH 38/42] pkg/exchange: add rate limiter for query ticker,
 account

---
 pkg/exchange/bitget/convert_test.go | 32 +++++++++++++++++++++++-
 pkg/exchange/bitget/exchange.go     | 38 ++++++++++++++++++++---------
 2 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go
index 6ab315eaa..5a80a045a 100644
--- a/pkg/exchange/bitget/convert_test.go
+++ b/pkg/exchange/bitget/convert_test.go
@@ -10,7 +10,37 @@ import (
 	"github.com/c9s/bbgo/pkg/types"
 )
 
-func TestToGlobalMarket(t *testing.T) {
+func Test_toGlobalBalance(t *testing.T) {
+	// sample:
+	// {
+	//        "coinId":"10012",
+	//        "coinName":"usdt",
+	//        "available":"0",
+	//        "frozen":"0",
+	//        "lock":"0",
+	//        "uTime":"1622697148"
+	//    }
+	asset := bitgetapi.AccountAsset{
+		CoinId:    2,
+		CoinName:  "USDT",
+		Available: fixedpoint.NewFromFloat(1.2),
+		Frozen:    fixedpoint.NewFromFloat(0.5),
+		Lock:      fixedpoint.NewFromFloat(0.5),
+		UTime:     types.NewMillisecondTimestampFromInt(1622697148),
+	}
+
+	assert.Equal(t, types.Balance{
+		Currency:          "USDT",
+		Available:         fixedpoint.NewFromFloat(1.2),
+		Locked:            fixedpoint.NewFromFloat(1), // frozen + lock
+		Borrowed:          fixedpoint.Zero,
+		Interest:          fixedpoint.Zero,
+		NetAsset:          fixedpoint.Zero,
+		MaxWithdrawAmount: fixedpoint.Zero,
+	}, toGlobalBalance(asset))
+}
+
+func Test_toGlobalMarket(t *testing.T) {
 	// sample:
 	//{
 	//            "symbol":"BTCUSDT_SPBL",
diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go
index b7af3828b..a3a570016 100644
--- a/pkg/exchange/bitget/exchange.go
+++ b/pkg/exchange/bitget/exchange.go
@@ -23,6 +23,10 @@ var log = logrus.WithFields(logrus.Fields{
 var (
 	// queryMarketRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-symbols
 	queryMarketRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5)
+	// queryAccountRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-account-assets
+	queryAccountRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5)
+	// queryTickerRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-single-ticker
+	queryTickerRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5)
 )
 
 type Exchange struct {
@@ -80,11 +84,15 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) {
 }
 
 func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticker, error) {
+	if err := queryTickerRateLimiter.Wait(ctx); err != nil {
+		return nil, fmt.Errorf("ticker rate limiter wait error: %w", err)
+	}
+
 	req := e.client.NewGetTickerRequest()
 	req.Symbol(symbol)
 	ticker, err := req.Do(ctx)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to query ticker: %w", err)
 	}
 
 	return &types.Ticker{
@@ -110,26 +118,34 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type
 }
 
 func (e *Exchange) QueryAccount(ctx context.Context) (*types.Account, error) {
-	req := e.client.NewGetAccountAssetsRequest()
-	resp, err := req.Do(ctx)
+	bals, err := e.QueryAccountBalances(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	bals := types.BalanceMap{}
-	for _, asset := range resp {
-		b := toGlobalBalance(asset)
-		bals[asset.CoinName] = b
-	}
-
 	account := types.NewAccount()
 	account.UpdateBalances(bals)
 	return account, nil
 }
 
 func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, error) {
-	// TODO implement me
-	panic("implement me")
+	if err := queryAccountRateLimiter.Wait(ctx); err != nil {
+		return nil, fmt.Errorf("account rate limiter wait error: %w", err)
+	}
+
+	req := e.client.NewGetAccountAssetsRequest()
+	resp, err := req.Do(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query account assets: %w", err)
+	}
+
+	bals := types.BalanceMap{}
+	for _, asset := range resp {
+		b := toGlobalBalance(asset)
+		bals[asset.CoinName] = b
+	}
+
+	return bals, nil
 }
 
 func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (createdOrder *types.Order, err error) {

From 470eb7dc0946a64c58eae9deaa5ad8860e5a0177 Mon Sep 17 00:00:00 2001
From: c9s <yoanlin93@gmail.com>
Date: Wed, 1 Nov 2023 15:22:53 +0800
Subject: [PATCH 39/42] cmd: skip reports for session has no trade

---
 pkg/cmd/backtest.go | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go
index 7ed612ee3..979412e9b 100644
--- a/pkg/cmd/backtest.go
+++ b/pkg/cmd/backtest.go
@@ -528,6 +528,11 @@ var BacktestCmd = &cobra.Command{
 
 		for _, session := range environ.Sessions() {
 			for symbol, trades := range session.Trades {
+				if len(trades.Trades) == 0 {
+					log.Warnf("session has no %s trades", symbol)
+					continue
+				}
+
 				tradeState := sessionTradeStats[session.Name][symbol]
 				profitFactor := tradeState.ProfitFactor
 				winningRatio := tradeState.WinningRatio
@@ -598,8 +603,11 @@ var BacktestCmd = &cobra.Command{
 	},
 }
 
-func createSymbolReport(userConfig *bbgo.Config, session *bbgo.ExchangeSession, symbol string, trades []types.Trade, intervalProfit *types.IntervalProfitCollector,
-	profitFactor, winningRatio fixedpoint.Value) (
+func createSymbolReport(
+	userConfig *bbgo.Config, session *bbgo.ExchangeSession, symbol string, trades []types.Trade,
+	intervalProfit *types.IntervalProfitCollector,
+	profitFactor, winningRatio fixedpoint.Value,
+) (
 	*backtest.SessionSymbolReport,
 	error,
 ) {
@@ -669,7 +677,10 @@ func createSymbolReport(userConfig *bbgo.Config, session *bbgo.ExchangeSession,
 	return &symbolReport, nil
 }
 
-func verify(userConfig *bbgo.Config, backtestService *service.BacktestService, sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time) error {
+func verify(
+	userConfig *bbgo.Config, backtestService *service.BacktestService,
+	sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time,
+) error {
 	for _, sourceExchange := range sourceExchanges {
 		err := backtestService.Verify(sourceExchange, userConfig.Backtest.Symbols, startTime, endTime)
 		if err != nil {
@@ -709,7 +720,10 @@ func getExchangeIntervals(ex types.Exchange) types.IntervalMap {
 	return types.SupportedIntervals
 }
 
-func sync(ctx context.Context, userConfig *bbgo.Config, backtestService *service.BacktestService, sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time) error {
+func sync(
+	ctx context.Context, userConfig *bbgo.Config, backtestService *service.BacktestService,
+	sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time,
+) error {
 	for _, symbol := range userConfig.Backtest.Symbols {
 		for _, sourceExchange := range sourceExchanges {
 			var supportIntervals = getExchangeIntervals(sourceExchange)

From 7a48d001a26aa64a3c122aafa575ef955fb3925e Mon Sep 17 00:00:00 2001
From: c9s <yoanlin93@gmail.com>
Date: Wed, 1 Nov 2023 15:23:27 +0800
Subject: [PATCH 40/42] backtest: return closed kline channel when empty symbol
 is given

---
 pkg/backtest/exchange.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/pkg/backtest/exchange.go b/pkg/backtest/exchange.go
index 851c4df14..28a6b49a4 100644
--- a/pkg/backtest/exchange.go
+++ b/pkg/backtest/exchange.go
@@ -382,6 +382,14 @@ func (e *Exchange) SubscribeMarketData(
 	}
 
 	log.Infof("querying klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals)
+	if len(symbols) == 0 {
+		log.Warnf("empty symbols, will not query kline data from the database")
+
+		c := make(chan types.KLine)
+		close(c)
+		return c, nil
+	}
+
 	klineC, errC := e.srv.QueryKLinesCh(startTime, endTime, e, symbols, intervals)
 	go func() {
 		if err := <-errC; err != nil {

From 00d4805321064c055a16bce34ec5716317ff1ec7 Mon Sep 17 00:00:00 2001
From: Edwin <edwin@am.is>
Date: Wed, 1 Nov 2023 16:14:21 +0800
Subject: [PATCH 41/42] pkg/exchange: add query tickers api

---
 pkg/exchange/bitget/convert.go      | 13 ++++++++
 pkg/exchange/bitget/convert_test.go | 49 +++++++++++++++++++++++++++++
 pkg/exchange/bitget/exchange.go     | 48 +++++++++++++++++++---------
 3 files changed, 96 insertions(+), 14 deletions(-)

diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go
index f40d1d58a..1339089fd 100644
--- a/pkg/exchange/bitget/convert.go
+++ b/pkg/exchange/bitget/convert.go
@@ -46,3 +46,16 @@ func toGlobalMarket(s bitgetapi.Symbol) types.Market {
 		MaxPrice:        fixedpoint.Zero,
 	}
 }
+
+func toGlobalTicker(ticker bitgetapi.Ticker) types.Ticker {
+	return types.Ticker{
+		Time:   ticker.Ts.Time(),
+		Volume: ticker.BaseVol,
+		Last:   ticker.Close,
+		Open:   ticker.OpenUtc0,
+		High:   ticker.High24H,
+		Low:    ticker.Low24H,
+		Buy:    ticker.BuyOne,
+		Sell:   ticker.SellOne,
+	}
+}
diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go
index 5a80a045a..770e4e5b8 100644
--- a/pkg/exchange/bitget/convert_test.go
+++ b/pkg/exchange/bitget/convert_test.go
@@ -94,3 +94,52 @@ func Test_toGlobalMarket(t *testing.T) {
 
 	assert.Equal(t, toGlobalMarket(inst), exp)
 }
+
+func Test_toGlobalTicker(t *testing.T) {
+	// sample:
+	// {
+	//        "symbol": "BTCUSDT",
+	//        "high24h": "24175.65",
+	//        "low24h": "23677.75",
+	//        "close": "24014.11",
+	//        "quoteVol": "177689342.3025",
+	//        "baseVol": "7421.5009",
+	//        "usdtVol": "177689342.302407",
+	//        "ts": "1660704288118",
+	//        "buyOne": "24013.94",
+	//        "sellOne": "24014.06",
+	//        "bidSz": "0.0663",
+	//        "askSz": "0.0119",
+	//        "openUtc0": "23856.72",
+	//        "changeUtc":"0.00301",
+	//        "change":"0.00069"
+	//    }
+	ticker := bitgetapi.Ticker{
+		Symbol:    "BTCUSDT",
+		High24H:   fixedpoint.NewFromFloat(24175.65),
+		Low24H:    fixedpoint.NewFromFloat(23677.75),
+		Close:     fixedpoint.NewFromFloat(24014.11),
+		QuoteVol:  fixedpoint.NewFromFloat(177689342.3025),
+		BaseVol:   fixedpoint.NewFromFloat(7421.5009),
+		UsdtVol:   fixedpoint.NewFromFloat(177689342.302407),
+		Ts:        types.NewMillisecondTimestampFromInt(1660704288118),
+		BuyOne:    fixedpoint.NewFromFloat(24013.94),
+		SellOne:   fixedpoint.NewFromFloat(24014.06),
+		BidSz:     fixedpoint.NewFromFloat(0.0663),
+		AskSz:     fixedpoint.NewFromFloat(0.0119),
+		OpenUtc0:  fixedpoint.NewFromFloat(23856.72),
+		ChangeUtc: fixedpoint.NewFromFloat(0.00301),
+		Change:    fixedpoint.NewFromFloat(0.00069),
+	}
+
+	assert.Equal(t, types.Ticker{
+		Time:   types.NewMillisecondTimestampFromInt(1660704288118).Time(),
+		Volume: fixedpoint.NewFromFloat(7421.5009),
+		Last:   fixedpoint.NewFromFloat(24014.11),
+		Open:   fixedpoint.NewFromFloat(23856.72),
+		High:   fixedpoint.NewFromFloat(24175.65),
+		Low:    fixedpoint.NewFromFloat(23677.75),
+		Buy:    fixedpoint.NewFromFloat(24013.94),
+		Sell:   fixedpoint.NewFromFloat(24014.06),
+	}, toGlobalTicker(ticker))
+}
diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go
index a3a570016..700bd2ea7 100644
--- a/pkg/exchange/bitget/exchange.go
+++ b/pkg/exchange/bitget/exchange.go
@@ -27,6 +27,8 @@ var (
 	queryAccountRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5)
 	// queryTickerRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-single-ticker
 	queryTickerRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5)
+	// queryTickersRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-all-tickers
+	queryTickersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5)
 )
 
 type Exchange struct {
@@ -90,26 +92,44 @@ func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticke
 
 	req := e.client.NewGetTickerRequest()
 	req.Symbol(symbol)
-	ticker, err := req.Do(ctx)
+	resp, err := req.Do(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to query ticker: %w", err)
 	}
 
-	return &types.Ticker{
-		Time:   ticker.Ts.Time(),
-		Volume: ticker.BaseVol,
-		Last:   ticker.Close,
-		Open:   ticker.OpenUtc0,
-		High:   ticker.High24H,
-		Low:    ticker.Low24H,
-		Buy:    ticker.BuyOne,
-		Sell:   ticker.SellOne,
-	}, nil
+	ticker := toGlobalTicker(*resp)
+	return &ticker, nil
 }
 
-func (e *Exchange) QueryTickers(ctx context.Context, symbol ...string) (map[string]types.Ticker, error) {
-	// TODO implement me
-	panic("implement me")
+func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[string]types.Ticker, error) {
+	tickers := map[string]types.Ticker{}
+	if len(symbols) > 0 {
+		for _, s := range symbols {
+			t, err := e.QueryTicker(ctx, s)
+			if err != nil {
+				return nil, err
+			}
+
+			tickers[s] = *t
+		}
+
+		return tickers, nil
+	}
+
+	if err := queryTickersRateLimiter.Wait(ctx); err != nil {
+		return nil, fmt.Errorf("tickers rate limiter wait error: %w", err)
+	}
+
+	resp, err := e.client.NewGetAllTickersRequest().Do(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query tickers: %w", err)
+	}
+
+	for _, s := range resp {
+		tickers[s.Symbol] = toGlobalTicker(s)
+	}
+
+	return tickers, nil
 }
 
 func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions) ([]types.KLine, error) {

From 9dc57f01cd54875bf1575f08fe81dc5b58e8eb8f Mon Sep 17 00:00:00 2001
From: c9s <yoanlin93@gmail.com>
Date: Wed, 1 Nov 2023 16:57:07 +0800
Subject: [PATCH 42/42] wall: refactor wall strategy with common.Strategy

---
 config/wall.yaml              |  10 +++-
 pkg/strategy/wall/strategy.go | 109 ++++++++--------------------------
 2 files changed, 34 insertions(+), 85 deletions(-)

diff --git a/config/wall.yaml b/config/wall.yaml
index 628088281..2f6104d3c 100644
--- a/config/wall.yaml
+++ b/config/wall.yaml
@@ -10,6 +10,14 @@ sessions:
     exchange: max
     envVarPrefix: MAX
 
+
+logging:
+  trade: true
+  order: true
+  # fields:
+  #   env: prod
+
+
 exchangeStrategies:
 
 - on: max
@@ -33,6 +41,6 @@ exchangeStrategies:
       byLayer:
         linear:
           domain: [ 1, 3 ]
-          range: [ 10.0, 30.0 ]
+          range: [ 10000.0, 30000.0 ]
 
 
diff --git a/pkg/strategy/wall/strategy.go b/pkg/strategy/wall/strategy.go
index 5cbb4294f..0967b241a 100644
--- a/pkg/strategy/wall/strategy.go
+++ b/pkg/strategy/wall/strategy.go
@@ -6,12 +6,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/c9s/bbgo/pkg/core"
-	"github.com/c9s/bbgo/pkg/util"
-
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
+	"github.com/c9s/bbgo/pkg/strategy/common"
+
 	"github.com/c9s/bbgo/pkg/bbgo"
 	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/types"
@@ -31,9 +30,10 @@ func init() {
 }
 
 type Strategy struct {
-	Environment          *bbgo.Environment
-	StandardIndicatorSet *bbgo.StandardIndicatorSet
-	Market               types.Market
+	*common.Strategy
+
+	Environment *bbgo.Environment
+	Market      types.Market
 
 	// Symbol is the market symbol you want to trade
 	Symbol string `json:"symbol"`
@@ -60,18 +60,8 @@ type Strategy struct {
 
 	session *bbgo.ExchangeSession
 
-	// persistence fields
-	Position    *types.Position    `json:"position,omitempty" persistence:"position"`
-	ProfitStats *types.ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"`
-
 	activeAdjustmentOrders *bbgo.ActiveOrderBook
 	activeWallOrders       *bbgo.ActiveOrderBook
-	orderStore             *core.OrderStore
-	tradeCollector         *core.TradeCollector
-
-	groupID uint32
-
-	stopC chan struct{}
 }
 
 func (s *Strategy) ID() string {
@@ -149,7 +139,6 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context, orderExecutor bbgo
 			Price:    askPrice,
 			Quantity: quantity,
 			Market:   s.Market,
-			GroupID:  s.groupID,
 		})
 
 	case types.SideTypeSell:
@@ -175,7 +164,6 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context, orderExecutor bbgo
 			Price:    bidPrice,
 			Quantity: quantity,
 			Market:   s.Market,
-			GroupID:  s.groupID,
 		})
 	}
 
@@ -189,12 +177,13 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context, orderExecutor bbgo
 		return err
 	}
 
-	s.orderStore.Add(createdOrders...)
 	s.activeAdjustmentOrders.Add(createdOrders...)
 	return nil
 }
 
 func (s *Strategy) placeWallOrders(ctx context.Context, orderExecutor bbgo.OrderExecutor) error {
+	log.Infof("placing wall orders...")
+
 	var submitOrders []types.SubmitOrder
 	var startPrice = s.FixedPrice
 	for i := 0; i < s.NumLayers; i++ {
@@ -217,7 +206,6 @@ func (s *Strategy) placeWallOrders(ctx context.Context, orderExecutor bbgo.Order
 			Price:    price,
 			Quantity: quantity,
 			Market:   s.Market,
-			GroupID:  s.groupID,
 		}
 		submitOrders = append(submitOrders, order)
 		switch s.Side {
@@ -240,74 +228,27 @@ func (s *Strategy) placeWallOrders(ctx context.Context, orderExecutor bbgo.Order
 		return err
 	}
 
-	s.orderStore.Add(createdOrders...)
+	log.Infof("wall orders placed: %+v", createdOrders)
+
 	s.activeWallOrders.Add(createdOrders...)
 	return err
 }
 
-func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error {
+func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error {
+	s.Strategy = &common.Strategy{}
+	s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID())
+
 	// initial required information
 	s.session = session
 
-	// calculate group id for orders
-	instanceID := s.InstanceID()
-	s.groupID = util.FNV32(instanceID)
-
-	// If position is nil, we need to allocate a new position for calculation
-	if s.Position == nil {
-		s.Position = types.NewPositionFromMarket(s.Market)
-	}
-
-	if s.ProfitStats == nil {
-		s.ProfitStats = types.NewProfitStats(s.Market)
-	}
-
-	// Always update the position fields
-	s.Position.Strategy = ID
-	s.Position.StrategyInstanceID = instanceID
-
-	s.stopC = make(chan struct{})
-
 	s.activeWallOrders = bbgo.NewActiveOrderBook(s.Symbol)
 	s.activeWallOrders.BindStream(session.UserDataStream)
 
 	s.activeAdjustmentOrders = bbgo.NewActiveOrderBook(s.Symbol)
 	s.activeAdjustmentOrders.BindStream(session.UserDataStream)
 
-	s.orderStore = core.NewOrderStore(s.Symbol)
-	s.orderStore.BindStream(session.UserDataStream)
-
-	s.tradeCollector = core.NewTradeCollector(s.Symbol, s.Position, s.orderStore)
-
-	s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) {
-		bbgo.Notify(trade)
-		s.ProfitStats.AddTrade(trade)
-
-		if profit.Compare(fixedpoint.Zero) == 0 {
-			s.Environment.RecordPosition(s.Position, trade, nil)
-		} else {
-			log.Infof("%s generated profit: %v", s.Symbol, profit)
-			p := s.Position.NewProfit(trade, profit, netProfit)
-			p.Strategy = ID
-			p.StrategyInstanceID = instanceID
-			bbgo.Notify(&p)
-
-			s.ProfitStats.AddProfit(p)
-			bbgo.Notify(&s.ProfitStats)
-
-			s.Environment.RecordPosition(s.Position, trade, &p)
-		}
-	})
-
-	s.tradeCollector.OnPositionUpdate(func(position *types.Position) {
-		log.Infof("position changed: %s", s.Position)
-		bbgo.Notify(s.Position)
-	})
-
-	s.tradeCollector.BindStream(session.UserDataStream)
-
 	session.UserDataStream.OnStart(func() {
-		if err := s.placeWallOrders(ctx, orderExecutor); err != nil {
+		if err := s.placeWallOrders(ctx, s.OrderExecutor); err != nil {
 			log.WithError(err).Errorf("can not place order")
 		}
 	})
@@ -318,9 +259,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se
 		}
 
 		// check if there is a canceled order had partially filled.
-		s.tradeCollector.Process()
+		s.OrderExecutor.TradeCollector().Process()
 
-		if err := s.placeAdjustmentOrders(ctx, orderExecutor); err != nil {
+		if err := s.placeAdjustmentOrders(ctx, s.OrderExecutor); err != nil {
 			log.WithError(err).Errorf("can not place order")
 		}
 	})
@@ -331,9 +272,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se
 		}
 
 		// check if there is a canceled order had partially filled.
-		s.tradeCollector.Process()
+		s.OrderExecutor.TradeCollector().Process()
 
-		if err := s.placeWallOrders(ctx, orderExecutor); err != nil {
+		if err := s.placeWallOrders(ctx, s.OrderExecutor); err != nil {
 			log.WithError(err).Errorf("can not place order")
 		}
 
@@ -342,9 +283,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se
 		}
 
 		// check if there is a canceled order had partially filled.
-		s.tradeCollector.Process()
+		s.OrderExecutor.TradeCollector().Process()
 
-		if err := s.placeAdjustmentOrders(ctx, orderExecutor); err != nil {
+		if err := s.placeAdjustmentOrders(ctx, s.OrderExecutor); err != nil {
 			log.WithError(err).Errorf("can not place order")
 		}
 	})
@@ -365,9 +306,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se
 					}
 
 					// check if there is a canceled order had partially filled.
-					s.tradeCollector.Process()
+					s.OrderExecutor.TradeCollector().Process()
 
-					if err := s.placeWallOrders(ctx, orderExecutor); err != nil {
+					if err := s.placeWallOrders(ctx, s.OrderExecutor); err != nil {
 						log.WithError(err).Errorf("can not place order")
 					}
 				}
@@ -377,7 +318,6 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se
 
 	bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) {
 		defer wg.Done()
-		close(s.stopC)
 
 		if err := s.activeWallOrders.GracefulCancel(ctx, s.session.Exchange); err != nil {
 			log.WithError(err).Errorf("graceful cancel order error")
@@ -387,7 +327,8 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se
 			log.WithError(err).Errorf("graceful cancel order error")
 		}
 
-		s.tradeCollector.Process()
+		// check if there is a canceled order had partially filled.
+		s.OrderExecutor.TradeCollector().Process()
 	})
 
 	return nil