From 90477826cf7b2623768ba73b0ee15faba9ee81de Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 17 Mar 2021 22:20:25 +0800 Subject: [PATCH 1/7] implement byte parser for fixedpoint parsing --- pkg/fixedpoint/convert.go | 59 +++++++++++++++++++++++++++++ pkg/fixedpoint/convert_test.go | 68 ++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 pkg/fixedpoint/convert_test.go diff --git a/pkg/fixedpoint/convert.go b/pkg/fixedpoint/convert.go index de868bf41..d9331021c 100644 --- a/pkg/fixedpoint/convert.go +++ b/pkg/fixedpoint/convert.go @@ -3,12 +3,14 @@ package fixedpoint import ( "database/sql/driver" "encoding/json" + "errors" "fmt" "math" "strconv" "sync/atomic" ) +const MaxPrecision = 12 const DefaultPrecision = 8 const DefaultPow = 1e8 @@ -64,6 +66,10 @@ func (v Value) Div(v2 Value) Value { return NewFromFloat(v.Float64() / v2.Float64()) } +func (v Value) Floor() Value { + return NewFromFloat(math.Floor(v.Float64())) +} + func (v Value) Sub(v2 Value) Value { return Value(int64(v) - int64(v2)) } @@ -156,6 +162,59 @@ func Must(v Value, err error) Value { return v } +var ErrPrecisionLoss = errors.New("precision loss") + +func Parse(input string) (num int64, numDecimalPoints int, err error) { + var neg int64 = 1 + var digit int64 + for i := 0 ; i < len(input) ; i++ { + c := input[i] + if c == '-' { + neg = -1 + } else if c >= '0' && c <= '9' { + digit, err = strconv.ParseInt(string(c), 10, 64) + if err != nil { + return + } + + num = num * 10 + digit + } else if c == '.' { + i++ + if i > len(input) - 1 { + err = fmt.Errorf("expect fraction numbers after dot") + return + } + + for j := i ; j < len(input); j++ { + fc := input[j] + if fc >= '0' && fc <= '9' { + digit, err = strconv.ParseInt(string(fc), 10, 64) + if err != nil { + return + } + + numDecimalPoints++ + num = num * 10 + digit + + if numDecimalPoints >= MaxPrecision { + return num, numDecimalPoints,ErrPrecisionLoss + } + } else { + err = fmt.Errorf("expect digit, got %c", fc) + return + } + } + break + } else { + err = fmt.Errorf("unexpected char %c", c) + return + } + } + + num = num * neg + return num, numDecimalPoints, nil +} + func NewFromString(input string) (Value, error) { v, err := strconv.ParseFloat(input, 64) if err != nil { diff --git a/pkg/fixedpoint/convert_test.go b/pkg/fixedpoint/convert_test.go new file mode 100644 index 000000000..4ecfc5397 --- /dev/null +++ b/pkg/fixedpoint/convert_test.go @@ -0,0 +1,68 @@ +package fixedpoint + +import "testing" + +func TestParse(t *testing.T) { + type args struct { + input string + } + tests := []struct { + name string + args args + wantNum int64 + wantNumDecimalPoints int + wantErr bool + }{ + { + args: args{ input: "-99.9" }, + wantNum: -999, + wantNumDecimalPoints: 1, + wantErr: false, + }, + { + args: args{ input: "0.12345678" }, + wantNum: 12345678, + wantNumDecimalPoints: 8, + wantErr: false, + }, + { + args: args{ input: "a" }, + wantNum: 0, + wantNumDecimalPoints: 0, + wantErr: true, + }, + { + args: args{ input: "0.1" }, + wantNum: 1, + wantNumDecimalPoints: 1, + wantErr: false, + }, + { + args: args{ input: "100" }, + wantNum: 100, + wantNumDecimalPoints: 0, + wantErr: false, + }, + { + args: args{ input: "100.9999" }, + wantNum: 1009999, + wantNumDecimalPoints: 4, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotNum, gotNumDecimalPoints, err := Parse(tt.args.input) + if (err != nil) != tt.wantErr { + t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotNum != tt.wantNum { + t.Errorf("Parse() gotNum = %v, want %v", gotNum, tt.wantNum) + } + if gotNumDecimalPoints != tt.wantNumDecimalPoints { + t.Errorf("Parse() gotNumDecimalPoints = %v, want %v", gotNumDecimalPoints, tt.wantNumDecimalPoints) + } + }) + } +} From 4a415a43b3ff2d76b7e1c0c5830dbc44c1acaada Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 18 Mar 2021 00:46:10 +0800 Subject: [PATCH 2/7] fix reward query --- pkg/exchange/batch/batch.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/exchange/batch/batch.go b/pkg/exchange/batch/batch.go index e18c83347..90733305d 100644 --- a/pkg/exchange/batch/batch.go +++ b/pkg/exchange/batch/batch.go @@ -232,10 +232,12 @@ func (q *RewardBatchQuery) Query(ctx context.Context, startTime, endTime time.Ti } c <- o - startTime = o.CreatedAt.Time() - lastID = o.UUID rewardKeys[o.UUID] = struct{}{} } + + end := len(rewards)-1 + startTime = rewards[end].CreatedAt.Time() + lastID = rewards[end].UUID } }() From 72c1f55b701bb2f5b3ba93808b353209256e00d0 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 18 Mar 2021 00:46:25 +0800 Subject: [PATCH 3/7] fix grid price calculation --- pkg/strategy/grid/strategy.go | 44 ++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/pkg/strategy/grid/strategy.go b/pkg/strategy/grid/strategy.go index 3ce39b6fc..669193ec6 100644 --- a/pkg/strategy/grid/strategy.go +++ b/pkg/strategy/grid/strategy.go @@ -31,7 +31,7 @@ type Snapshot struct { Orders []types.SubmitOrder `json:"orders,omitempty"` FilledBuyGrids map[fixedpoint.Value]struct{} `json:"filledBuyGrids"` FilledSellGrids map[fixedpoint.Value]struct{} `json:"filledSellGrids"` - Position *bbgo.Position `json:"position,omitempty"` + Position *bbgo.Position `json:"position,omitempty"` } type Strategy struct { @@ -113,14 +113,18 @@ func (s *Strategy) generateGridSellOrders(session *bbgo.ExchangeSession) ([]type } currentPrice := fixedpoint.NewFromFloat(currentPriceFloat) - priceRange := s.UpperPrice - s.LowerPrice - if priceRange <= 0 { - return nil, fmt.Errorf("upper price %f should not be less than or equal to lower price %f", s.UpperPrice.Float64(), s.LowerPrice.Float64()) + if currentPrice > s.UpperPrice { + return nil, fmt.Errorf("current price %f is higher than upper price %f", currentPrice.Float64(), s.UpperPrice.Float64()) } + priceRange := s.UpperPrice - s.LowerPrice numGrids := fixedpoint.NewFromInt(s.GridNum) gridSpread := priceRange.Div(numGrids) - startPrice := fixedpoint.Max(s.LowerPrice, currentPrice+gridSpread) + + // find the nearest grid price from the current price + startPrice := fixedpoint.Max( + s.LowerPrice, + s.UpperPrice-(s.UpperPrice-currentPrice).Div(gridSpread).Floor().Mul(gridSpread)) if startPrice > s.UpperPrice { return nil, fmt.Errorf("current price %f exceeded the upper price boundary %f", @@ -196,14 +200,24 @@ func (s *Strategy) generateGridBuyOrders(session *bbgo.ExchangeSession) ([]types } currentPrice := fixedpoint.NewFromFloat(currentPriceFloat) - priceRange := s.UpperPrice - s.LowerPrice - if priceRange <= 0 { - return nil, fmt.Errorf("upper price %f should not be less than or equal to lower price %f", s.UpperPrice.Float64(), s.LowerPrice.Float64()) + if currentPrice < s.LowerPrice { + return nil, fmt.Errorf("current price %f is lower than the lower price %f", currentPrice.Float64(), s.LowerPrice.Float64()) } + priceRange := s.UpperPrice - s.LowerPrice numGrids := fixedpoint.NewFromInt(s.GridNum) gridSpread := priceRange.Div(numGrids) - startPrice := fixedpoint.Min(s.UpperPrice, currentPrice-gridSpread) + + // Find the nearest grid price for placing buy orders: + // buyRange = currentPrice - lowerPrice + // numOfBuyGrids = Floor(buyRange / gridSpread) + // startPrice = lowerPrice + numOfBuyGrids * gridSpread + // priceOfBuyOrder1 = startPrice + // priceOfBuyOrder2 = startPrice - gridSpread + // priceOfBuyOrder3 = startPrice - gridSpread * 2 + startPrice := fixedpoint.Min( + s.UpperPrice, + s.LowerPrice+(currentPrice-s.LowerPrice).Div(gridSpread).Floor().Mul(gridSpread)) if startPrice < s.LowerPrice { return nil, fmt.Errorf("current price %f exceeded the lower price boundary %f", @@ -417,8 +431,16 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se s.Side = types.SideTypeBoth } + if s.UpperPrice == 0 { + return errors.New("upperPrice can not be zero, you forgot to set?") + } + + if s.LowerPrice == 0 { + return errors.New("lowerPrice can not be zero, you forgot to set?") + } + if s.UpperPrice <= s.LowerPrice { - return fmt.Errorf("upper price (%f) should not be less than lower price (%f)", s.UpperPrice.Float64(), s.LowerPrice.Float64()) + return fmt.Errorf("upperPrice (%f) should not be less than or equal to lowerPrice (%f)", s.UpperPrice.Float64(), s.LowerPrice.Float64()) } var snapshot Snapshot @@ -465,7 +487,7 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se log.Infof("backing up active orders...") submitOrders := s.activeOrders.Backup() snapshot := Snapshot{ - Orders: submitOrders, + Orders: submitOrders, Position: &s.position, } From 8d784576cd32ba95fa61283fbb0d84d9a1bb3689 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 18 Mar 2021 01:14:56 +0800 Subject: [PATCH 4/7] put state vars into the state struct for persistence --- pkg/strategy/grid/strategy.go | 78 +++++++++++++++-------------------- 1 file changed, 33 insertions(+), 45 deletions(-) diff --git a/pkg/strategy/grid/strategy.go b/pkg/strategy/grid/strategy.go index 669193ec6..bcd6d1814 100644 --- a/pkg/strategy/grid/strategy.go +++ b/pkg/strategy/grid/strategy.go @@ -26,8 +26,8 @@ func init() { bbgo.RegisterStrategy(ID, &Strategy{}) } -// Snapshot is the grid snapshot -type Snapshot struct { +// State is the grid snapshot +type State struct { Orders []types.SubmitOrder `json:"orders,omitempty"` FilledBuyGrids map[fixedpoint.Value]struct{} `json:"filledBuyGrids"` FilledSellGrids map[fixedpoint.Value]struct{} `json:"filledSellGrids"` @@ -84,8 +84,7 @@ type Strategy struct { // Long means you want to hold more base asset than the quote asset. Long bool `json:"long,omitempty" yaml:"long,omitempty"` - filledBuyGrids map[fixedpoint.Value]struct{} - filledSellGrids map[fixedpoint.Value]struct{} + state *State // orderStore is used to store all the created orders, so that we can filter the trades. orderStore *bbgo.OrderStore @@ -93,8 +92,6 @@ type Strategy struct { // activeOrders is the locally maintained active order book of the maker orders. activeOrders *bbgo.LocalActiveOrderBook - position bbgo.Position - // any created orders for tracking trades orders map[uint64]types.Order @@ -169,7 +166,7 @@ func (s *Strategy) generateGridSellOrders(session *bbgo.ExchangeSession) ([]type baseBalance.Available.Float64()) } - if _, filled := s.filledSellGrids[price]; filled { + if _, filled := s.state.FilledSellGrids[price]; filled { log.Debugf("sell grid at price %f is already filled, skipping", price.Float64()) continue } @@ -186,7 +183,7 @@ func (s *Strategy) generateGridSellOrders(session *bbgo.ExchangeSession) ([]type }) baseBalance.Available -= quantity - s.filledSellGrids[price] = struct{}{} + s.state.FilledSellGrids[price] = struct{}{} } return orders, nil @@ -236,7 +233,7 @@ func (s *Strategy) generateGridBuyOrders(session *bbgo.ExchangeSession) ([]types } log.Infof("placing grid buy orders from %f to %f, grid spread %f", - (currentPrice - gridSpread).Float64(), + startPrice.Float64(), s.LowerPrice.Float64(), gridSpread.Float64()) @@ -263,7 +260,7 @@ func (s *Strategy) generateGridBuyOrders(session *bbgo.ExchangeSession) ([]types quoteQuantity.Float64()) } - if _, filled := s.filledBuyGrids[price]; filled { + if _, filled := s.state.FilledBuyGrids[price]; filled { log.Debugf("buy grid at price %f is already filled, skipping", price.Float64()) continue } @@ -280,7 +277,7 @@ func (s *Strategy) generateGridBuyOrders(session *bbgo.ExchangeSession) ([]types }) balance.Available -= quoteQuantity - s.filledBuyGrids[price] = struct{}{} + s.state.FilledBuyGrids[price] = struct{}{} } return orders, nil @@ -323,7 +320,7 @@ func (s *Strategy) placeGridBuyOrders(orderExecutor bbgo.OrderExecutor, session } func (s *Strategy) placeGridOrders(orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) { - log.Infof("placing grid orders...") + log.Infof("placing grid orders on side %s...", s.Side) switch s.Side { @@ -367,7 +364,7 @@ func (s *Strategy) tradeUpdateHandler(trade types.Trade) { return } - profit, madeProfit := s.position.AddTrade(trade) + profit, madeProfit := s.state.Position.AddTrade(trade) if madeProfit { s.Notify("profit: %f", profit.Float64()) } @@ -443,30 +440,34 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se return fmt.Errorf("upperPrice (%f) should not be less than or equal to lowerPrice (%f)", s.UpperPrice.Float64(), s.LowerPrice.Float64()) } - var snapshot Snapshot - var snapshotLoaded = false + var stateLoaded = false if s.Persistence != nil { - if err := s.Persistence.Load(&snapshot, ID, s.Symbol, "snapshot"); err != nil { + var state State + if err := s.Persistence.Load(&state, ID, s.Symbol, "state"); err != nil { if err != service.ErrPersistenceNotExists { - return errors.Wrapf(err, "snapshot load error") + return errors.Wrapf(err, "state load error") } } else { - log.Infof("active order snapshot loaded") - snapshotLoaded = true + log.Infof("grid state loaded") + stateLoaded = true + s.state = &state } } - s.filledBuyGrids = make(map[fixedpoint.Value]struct{}) - s.filledSellGrids = make(map[fixedpoint.Value]struct{}) + if s.state == nil { + position, ok := session.Position(s.Symbol) + if !ok { + return fmt.Errorf("position not found") + } - position, ok := session.Position(s.Symbol) - if !ok { - return fmt.Errorf("position not found") + s.state = &State{ + FilledBuyGrids: make(map[fixedpoint.Value]struct{}), + FilledSellGrids: make(map[fixedpoint.Value]struct{}), + Position: position, + } } - s.position = *position - - s.Notify("current position %+v", position) + s.Notify("current position %+v", s.state.Position) instanceID := fmt.Sprintf("grid-%s-%d", s.Symbol, s.GridNum) s.groupID = generateGroupID(instanceID) @@ -484,14 +485,10 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se defer wg.Done() if s.Persistence != nil { - log.Infof("backing up active orders...") + log.Infof("backing up grid state...") submitOrders := s.activeOrders.Backup() - snapshot := Snapshot{ - Orders: submitOrders, - Position: &s.position, - } - - if err := s.Persistence.Save(&snapshot, ID, s.Symbol, "snapshot"); err != nil { + s.state.Orders = submitOrders + if err := s.Persistence.Save(s.state, ID, s.Symbol, "snapshot"); err != nil { log.WithError(err).Error("can not save active order backups") } else { log.Infof("active order snapshot saved") @@ -514,22 +511,13 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se session.Stream.OnTradeUpdate(s.tradeUpdateHandler) session.Stream.OnStart(func() { - if snapshotLoaded && len(snapshot.Orders) > 0 { - createdOrders, err := orderExecutor.SubmitOrders(ctx, snapshot.Orders...) + if stateLoaded && len(s.state.Orders) > 0 { + createdOrders, err := orderExecutor.SubmitOrders(ctx, s.state.Orders...) if err != nil { log.WithError(err).Error("active orders restore error") } s.activeOrders.Add(createdOrders...) s.orderStore.Add(createdOrders...) - if snapshot.FilledSellGrids != nil { - s.filledSellGrids = snapshot.FilledSellGrids - } - if snapshot.FilledBuyGrids != nil { - s.filledBuyGrids = snapshot.FilledBuyGrids - } - if snapshot.Position != nil { - s.position = *snapshot.Position - } } else { s.placeGridOrders(orderExecutor, session) } From 85b6cb81a2c01fe9efecc176b837e6ef62883ceb Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 18 Mar 2021 01:15:06 +0800 Subject: [PATCH 5/7] make local active orderbook json marshallable --- pkg/bbgo/active_book.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/bbgo/active_book.go b/pkg/bbgo/active_book.go index 662b60eb5..9f1a895b4 100644 --- a/pkg/bbgo/active_book.go +++ b/pkg/bbgo/active_book.go @@ -1,6 +1,8 @@ package bbgo import ( + "encoding/json" + log "github.com/sirupsen/logrus" "github.com/c9s/bbgo/pkg/types" @@ -22,6 +24,11 @@ func NewLocalActiveOrderBook() *LocalActiveOrderBook { } } +func (b *LocalActiveOrderBook) MarshalJSON() ([]byte, error) { + orders := b.Backup() + return json.Marshal(orders) +} + func (b *LocalActiveOrderBook) Backup() []types.SubmitOrder { return append(b.Bids.Backup(), b.Asks.Backup()...) } From dd87bde7856ad8de9a79cd8a809e4de8c938043b Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 18 Mar 2021 01:15:29 +0800 Subject: [PATCH 6/7] fix reward sync time range issue --- pkg/exchange/batch/batch.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/exchange/batch/batch.go b/pkg/exchange/batch/batch.go index 90733305d..de9216130 100644 --- a/pkg/exchange/batch/batch.go +++ b/pkg/exchange/batch/batch.go @@ -221,6 +221,7 @@ func (q *RewardBatchQuery) Query(ctx context.Context, startTime, endTime time.Ti return } + newCnt := 0 for _, o := range rewards { if _, ok := rewardKeys[o.UUID]; ok { continue @@ -231,11 +232,16 @@ func (q *RewardBatchQuery) Query(ctx context.Context, startTime, endTime time.Ti return } + newCnt++ c <- o rewardKeys[o.UUID] = struct{}{} } - end := len(rewards)-1 + if newCnt == 0 { + return + } + + end := len(rewards) - 1 startTime = rewards[end].CreatedAt.Time() lastID = rewards[end].UUID } From cad8349a1aeccb516d13c286d14ed8fb76a360ed Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 18 Mar 2021 01:15:49 +0800 Subject: [PATCH 7/7] remove state OrderStateFinalizing from the order state since we are only interested in the closed orders --- pkg/exchange/max/maxapi/order.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/exchange/max/maxapi/order.go b/pkg/exchange/max/maxapi/order.go index ea5900a10..2b21089ca 100644 --- a/pkg/exchange/max/maxapi/order.go +++ b/pkg/exchange/max/maxapi/order.go @@ -72,7 +72,7 @@ type Order struct { func (s *OrderService) Closed(market string, options QueryOrderOptions) ([]Order, error) { payload := map[string]interface{}{ "market": market, - "state": []OrderState{OrderStateFinalizing, OrderStateDone, OrderStateCancel, OrderStateFailed}, + "state": []OrderState{OrderStateDone, OrderStateCancel, OrderStateFailed}, "order_by": "desc", "pagination": false, }