fix and improve backtest

pkg/backtest/exchange.go

@@ -52,13 +52,13 @@ func NewExchange(sourceName types.ExchangeName, srv *service.BacktestService, co
 		panic(err)
 	}
 
-	balances := config.Account.Balances.BalanceMap()
-
 	account := &types.Account{
 		MakerCommission: config.Account.MakerCommission,
 		TakerCommission: config.Account.TakerCommission,
 		AccountType:     "SPOT", // currently not used
 	}
+
+	balances := config.Account.Balances.BalanceMap()
 	account.UpdateBalances(balances)
 
 	e := &Exchange{
@@ -93,19 +93,18 @@ func (e *Exchange) NewStream() types.Stream {
 		e.trades[trade.Symbol] = append(e.trades[trade.Symbol], trade)
 	})
 
-	for _, symbol := range e.config.Symbols {
+	for symbol, market := range e.markets {
-		market, ok := e.markets[symbol]
+		matching := &SimplePriceMatching{
-		if !ok {
-			panic(fmt.Errorf("market %s is undefined", symbol))
-		}
-
-		e.matchingBooks[symbol] = &SimplePriceMatching{
 			CurrentTime:     e.startTime,
 			Account:         e.account,
 			Market:          market,
 			MakerCommission: e.config.Account.MakerCommission,
 			TakerCommission: e.config.Account.TakerCommission,
 		}
+		matching.OnTradeUpdate(e.stream.EmitTradeUpdate)
+		matching.OnOrderUpdate(e.stream.EmitOrderUpdate)
+		matching.OnBalanceUpdate(e.stream.EmitBalanceUpdate)
+		e.matchingBooks[symbol] = matching
 	}
 
 	return e.stream
@@ -119,7 +118,6 @@ func (e Exchange) SubmitOrders(ctx context.Context, orders ...types.SubmitOrder)
 			return nil, fmt.Errorf("matching engine is not initialized for symbol %s", symbol)
 		}
 
-
 		createdOrder, trade, err := matching.PlaceOrder(order)
 		if err != nil {
 			return nil, err

pkg/backtest/matching.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 
 	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/types"
@@ -49,7 +50,7 @@ type SimplePriceMatching struct {
 
 	tradeUpdateCallbacks   []func(trade types.Trade)
 	orderUpdateCallbacks   []func(order types.Order)
-	accountUpdateCallbacks []func(balances types.BalanceMap)
+	balanceUpdateCallbacks []func(balances types.BalanceMap)
 }
 
 func (m *SimplePriceMatching) CancelOrder(o types.Order) (types.Order, error) {
@@ -73,43 +74,43 @@ func (m *SimplePriceMatching) CancelOrder(o types.Order) (types.Order, error) {
 	case types.SideTypeSell:
 		m.mu.Lock()
 		var orders []types.Order
-		for _, order := range m.bidOrders {
+		for _, order := range m.askOrders {
 			if o.OrderID == order.OrderID {
 				found = true
 				continue
 			}
 			orders = append(orders, order)
 		}
-		m.bidOrders = orders
+		m.askOrders = orders
 		m.mu.Unlock()
 
 	}
 
 	if !found {
+		logrus.Panicf("cancel order failed, order %d not found: %+v", o.OrderID, o)
+
 		return o, fmt.Errorf("cancel order failed, order %d not found: %+v", o.OrderID, o)
 	}
 
 	switch o.Side {
 	case types.SideTypeBuy:
-		if err := m.Account.UnlockBalance(m.Market.QuoteCurrency, o.Price*o.Quantity); err != nil {
+		if err := m.Account.UnlockBalance(m.Market.QuoteCurrency, fixedpoint.NewFromFloat(o.Price*o.Quantity)); err != nil {
 			return o, err
 		}
 
 	case types.SideTypeSell:
-		if err := m.Account.UnlockBalance(m.Market.BaseCurrency, o.Quantity); err != nil {
+		if err := m.Account.UnlockBalance(m.Market.BaseCurrency, fixedpoint.NewFromFloat(o.Quantity)); err != nil {
 			return o, err
 		}
 	}
 
 	o.Status = types.OrderStatusCanceled
 	m.EmitOrderUpdate(o)
-	m.EmitAccountUpdate(m.Account.Balances())
+	m.EmitBalanceUpdate(m.Account.Balances())
 	return o, nil
 }
 
 func (m *SimplePriceMatching) PlaceOrder(o types.SubmitOrder) (closedOrders *types.Order, trades *types.Trade, err error) {
-	// start from one
-	orderID := incOrderID()
 
 	// price for checking account balance
 	price := o.Price
@@ -123,19 +124,21 @@ func (m *SimplePriceMatching) PlaceOrder(o types.SubmitOrder) (closedOrders *typ
 	switch o.Side {
 	case types.SideTypeBuy:
 		quote := price * o.Quantity
-		if err := m.Account.LockBalance(m.Market.QuoteCurrency, quote); err != nil {
+		if err := m.Account.LockBalance(m.Market.QuoteCurrency, fixedpoint.NewFromFloat(quote)); err != nil {
 			return nil, nil, err
 		}
 
 	case types.SideTypeSell:
 		baseQuantity := o.Quantity
-		if err := m.Account.LockBalance(m.Market.BaseCurrency, baseQuantity); err != nil {
+		if err := m.Account.LockBalance(m.Market.BaseCurrency, fixedpoint.NewFromFloat(baseQuantity)); err != nil {
 			return nil, nil, err
 		}
 	}
 
-	m.EmitAccountUpdate(m.Account.Balances())
+	m.EmitBalanceUpdate(m.Account.Balances())
 
+	// start from one
+	orderID := incOrderID()
 	order := m.newOrder(o, orderID)
 
 	if o.Type == types.OrderTypeMarket {
@@ -150,7 +153,7 @@ func (m *SimplePriceMatching) PlaceOrder(o types.SubmitOrder) (closedOrders *typ
 		order.ExecutedQuantity = order.Quantity
 		order.Price = price
 		m.EmitOrderUpdate(order)
-		m.EmitAccountUpdate(m.Account.Balances())
+		m.EmitBalanceUpdate(m.Account.Balances())
 		return &order, &trade, nil
 	}
 
@@ -177,10 +180,13 @@ func (m *SimplePriceMatching) executeTrade(trade types.Trade) {
 	var err error
 	// execute trade, update account balances
 	if trade.IsBuyer {
-		quote := trade.Price * trade.Quantity
+		err = m.Account.UseLockedBalance(m.Market.QuoteCurrency, fixedpoint.NewFromFloat(trade.Price*trade.Quantity))
-		err = m.Account.UseLockedBalance(m.Market.QuoteCurrency, quote)
+
+		_ = m.Account.AddBalance(m.Market.BaseCurrency, fixedpoint.NewFromFloat(trade.Quantity))
 	} else {
-		err = m.Account.UseLockedBalance(m.Market.BaseCurrency, trade.Quantity)
+		err = m.Account.UseLockedBalance(m.Market.BaseCurrency, fixedpoint.NewFromFloat(trade.Quantity))
+
+		_ = m.Account.AddBalance(m.Market.QuoteCurrency, fixedpoint.NewFromFloat(trade.Quantity*trade.Price))
 	}
 
 	if err != nil {
@@ -188,7 +194,7 @@ func (m *SimplePriceMatching) executeTrade(trade types.Trade) {
 	}
 
 	m.EmitTradeUpdate(trade)
-	m.EmitAccountUpdate(m.Account.Balances())
+	m.EmitBalanceUpdate(m.Account.Balances())
 	return
 }
 
@@ -391,69 +397,34 @@ func (m *SimplePriceMatching) SellToPrice(price fixedpoint.Value) (closedOrders
 	return closedOrders, trades
 }
 
-func emitTxn(stream *Stream, trades []types.Trade, orders []types.Order) {
+func (m *SimplePriceMatching) processKLine(kline types.KLine) {
-	for _, t := range trades {
-		stream.EmitTradeUpdate(t)
-	}
-	for _, o := range orders {
-		stream.EmitOrderUpdate(o)
-	}
-}
-func (m *SimplePriceMatching) processKLine(stream *Stream, kline types.KLine) {
 	m.CurrentTime = kline.EndTime
 
 	switch kline.GetTrend() {
 	case types.TrendDown:
 		if kline.High > kline.Open {
-			orders, trades := m.BuyToPrice(fixedpoint.NewFromFloat(kline.High))
+			m.BuyToPrice(fixedpoint.NewFromFloat(kline.High))
-			emitTxn(stream, trades, orders)
 		}
 
 		if kline.Low > kline.Close {
-			orders, trades := m.SellToPrice(fixedpoint.NewFromFloat(kline.Low))
+			m.SellToPrice(fixedpoint.NewFromFloat(kline.Low))
-			emitTxn(stream, trades, orders)
 		}
-		orders, trades := m.SellToPrice(fixedpoint.NewFromFloat(kline.Close))
+
-		emitTxn(stream, trades, orders)
+		m.SellToPrice(fixedpoint.NewFromFloat(kline.Close))
 
 	case types.TrendUp:
 		if kline.Low < kline.Open {
-			orders, trades := m.SellToPrice(fixedpoint.NewFromFloat(kline.Low))
+			m.SellToPrice(fixedpoint.NewFromFloat(kline.Low))
-			emitTxn(stream, trades, orders)
 		}
 
 		if kline.High > kline.Close {
-			orders, trades := m.BuyToPrice(fixedpoint.NewFromFloat(kline.High))
+			m.BuyToPrice(fixedpoint.NewFromFloat(kline.High))
-			emitTxn(stream, trades, orders)
 		}
-		orders, trades := m.BuyToPrice(fixedpoint.NewFromFloat(kline.Close))
+
-		emitTxn(stream, trades, orders)
+		m.BuyToPrice(fixedpoint.NewFromFloat(kline.Close))
 	}
 }
 
-type Matching struct {
-	Symbol string
-	Asks   PriceOrderSlice
-	Bids   PriceOrderSlice
-
-	OrderID     uint64
-	CurrentTime time.Time
-}
-
-func (m *Matching) PlaceOrder(o types.SubmitOrder) {
-	var order = types.Order{
-		SubmitOrder:      o,
-		Exchange:         "backtest",
-		OrderID:          m.OrderID,
-		Status:           types.OrderStatusNew,
-		ExecutedQuantity: 0,
-		IsWorking:        false,
-		CreationTime:     m.CurrentTime,
-		UpdateTime:       m.CurrentTime,
-	}
-	_ = order
-}
-
 func (m *SimplePriceMatching) newOrder(o types.SubmitOrder, orderID uint64) types.Order {
 	return types.Order{
 		OrderID:          orderID,
@@ -461,7 +432,7 @@ func (m *SimplePriceMatching) newOrder(o types.SubmitOrder, orderID uint64) type
 		Exchange:         "backtest",
 		Status:           types.OrderStatusNew,
 		ExecutedQuantity: 0,
-		IsWorking:        false,
+		IsWorking:        true,
 		CreationTime:     m.CurrentTime,
 		UpdateTime:       m.CurrentTime,
 	}

pkg/backtest/matching_test.go

@@ -28,8 +28,8 @@ func TestSimplePriceMatching_LimitOrder(t *testing.T) {
 	}
 
 	account.UpdateBalances(types.BalanceMap{
-		"USDT": {Currency: "USDT", Available: 1000000.0},
+		"USDT": {Currency: "USDT", Available: fixedpoint.NewFromFloat(1000000.0)},
-		"BTC":  {Currency: "BTC", Available: 100.0},
+		"BTC":  {Currency: "BTC", Available: fixedpoint.NewFromFloat(100.0)},
 	})
 
 	market := types.Market{
@@ -71,6 +71,10 @@ func TestSimplePriceMatching_LimitOrder(t *testing.T) {
 	closedOrders, trades = engine.SellToPrice(fixedpoint.NewFromFloat(8000.0))
 	assert.Len(t, closedOrders, 1)
 	assert.Len(t, trades, 1)
+	for _, trade := range trades {
+		assert.True(t, trade.IsBuyer)
+	}
+
 	for _, o := range closedOrders {
 		assert.Equal(t, types.SideTypeBuy, o.Side)
 	}
@@ -89,7 +93,6 @@ func TestSimplePriceMatching_LimitOrder(t *testing.T) {
 	for _, o := range closedOrders {
 		assert.Equal(t, types.SideTypeSell, o.Side)
 	}
-
 	for _, trade := range trades {
 		assert.Equal(t, types.SideTypeSell, trade.Side)
 	}

pkg/backtest/simplepricematching_callbacks.go

@@ -26,12 +26,12 @@ func (m *SimplePriceMatching) EmitOrderUpdate(order types.Order) {
 	}
 }
 
-func (m *SimplePriceMatching) OnAccountUpdate(cb func(balances types.BalanceMap)) {
+func (m *SimplePriceMatching) OnBalanceUpdate(cb func(balances types.BalanceMap)) {
-	m.accountUpdateCallbacks = append(m.accountUpdateCallbacks, cb)
+	m.balanceUpdateCallbacks = append(m.balanceUpdateCallbacks, cb)
 }
 
-func (m *SimplePriceMatching) EmitAccountUpdate(balances types.BalanceMap) {
+func (m *SimplePriceMatching) EmitBalanceUpdate(balances types.BalanceMap) {
-	for _, cb := range m.accountUpdateCallbacks {
+	for _, cb := range m.balanceUpdateCallbacks {
 		cb(balances)
 	}
 }

pkg/backtest/stream.go

@@ -21,7 +21,8 @@ func (s *Stream) Connect(ctx context.Context) error {
 	loadedSymbols := map[string]struct{}{}
 	loadedIntervals := map[types.Interval]struct{}{
 		// 1m interval is required for the backtest matching engine
-		types.Interval1m: struct{}{},
+		types.Interval1m: {},
+		types.Interval1d: {},
 	}
 
 	for _, sub := range s.Subscriptions {
@@ -56,7 +57,7 @@ func (s *Stream) Connect(ctx context.Context) error {
 				if !ok {
 					log.Errorf("matching book of %s is not initialized", k.Symbol)
 				}
-				matching.processKLine(s, k)
+				matching.processKLine(k)
 			}
 
 			s.EmitKLineClosed(k)

pkg/types/account.go

@@ -6,17 +6,27 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"github.com/c9s/bbgo/pkg/util"
+	"github.com/c9s/bbgo/pkg/fixedpoint"
 )
 
 type Balance struct {
-	Currency  string  `json:"currency"`
+	Currency  string           `json:"currency"`
-	Available float64 `json:"available"`
+	Available fixedpoint.Value `json:"available"`
-	Locked    float64 `json:"locked"`
+	Locked    fixedpoint.Value `json:"locked"`
 }
 
 type BalanceMap map[string]Balance
 
+func (m BalanceMap) Print() {
+	for _, balance := range m {
+		if balance.Locked > 0 {
+			logrus.Infof(" %s: %f (locked %f)", balance.Currency, balance.Available.Float64(), balance.Locked.Float64())
+		} else {
+			logrus.Infof(" %s: %f", balance.Currency, balance.Available.Float64())
+		}
+	}
+}
+
 type Account struct {
 	sync.Mutex
 
@@ -53,7 +63,7 @@ func (a *Account) Balance(currency string) (balance Balance, ok bool) {
 	return balance, ok
 }
 
-func (a *Account) AddBalance(currency string, fund float64) error {
+func (a *Account) AddBalance(currency string, fund fixedpoint.Value) error {
 	a.Lock()
 	defer a.Unlock()
 
@@ -72,7 +82,7 @@ func (a *Account) AddBalance(currency string, fund float64) error {
 	return nil
 }
 
-func (a *Account) UseLockedBalance(currency string, fund float64) error {
+func (a *Account) UseLockedBalance(currency string, fund fixedpoint.Value) error {
 	a.Lock()
 	defer a.Unlock()
 
@@ -83,24 +93,29 @@ func (a *Account) UseLockedBalance(currency string, fund float64) error {
 		return nil
 	}
 
-	return fmt.Errorf("trying to use more than locked: locked %f < want to use %f", balance.Locked, fund)
+	return fmt.Errorf("trying to use more than locked: locked %f < want to use %f", balance.Locked.Float64(), fund.Float64())
 }
 
-func (a *Account) UnlockBalance(currency string, unlocked float64) error {
+func (a *Account) UnlockBalance(currency string, unlocked fixedpoint.Value) error {
 	a.Lock()
 	defer a.Unlock()
+
 	balance, ok := a.balances[currency]
-	if ok && balance.Locked >= unlocked {
+	if !ok {
-		balance.Locked -= unlocked
+		return fmt.Errorf("trying to unlocked inexisted balance: %s", currency)
-		balance.Available += unlocked
-		a.balances[currency] = balance
-		return nil
 	}
 
-	return fmt.Errorf("trying to unlocked more than locked: locked %f < want to unlock %f", balance.Locked, unlocked)
+	if unlocked > balance.Locked {
+		return fmt.Errorf("trying to unlocked more than locked %s: locked %f < want to unlock %f", currency, balance.Locked.Float64(), unlocked.Float64())
+	}
+
+	balance.Locked -= unlocked
+	balance.Available += unlocked
+	a.balances[currency] = balance
+	return nil
 }
 
-func (a *Account) LockBalance(currency string, locked float64) error {
+func (a *Account) LockBalance(currency string, locked fixedpoint.Value) error {
 	a.Lock()
 	defer a.Unlock()
 
@@ -112,10 +127,10 @@ func (a *Account) LockBalance(currency string, locked float64) error {
 		return nil
 	}
 
-	return fmt.Errorf("insufficient available balance for lock %f", locked)
+	return fmt.Errorf("insufficient available balance %s for lock: want to lock %f, available %f", currency, locked.Float64(), balance.Available.Float64())
 }
 
-func (a *Account) UpdateBalances(balances map[string]Balance) {
+func (a *Account) UpdateBalances(balances BalanceMap) {
 	a.Lock()
 	defer a.Unlock()
 
@@ -138,8 +153,8 @@ func (a *Account) Print() {
 	defer a.Unlock()
 
 	for _, balance := range a.balances {
-		if util.NotZero(balance.Available) {
+		if balance.Available != 0 {
-			logrus.Infof("account balance %s %f", balance.Currency, balance.Available)
+			logrus.Infof("account balance %s %f", balance.Currency, balance.Available.Float64())
 		}
 	}
 }

pkg/types/account_test.go

@@ -4,58 +4,59 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/c9s/bbgo/pkg/fixedpoint"
 )
 
 func TestAccountLockAndUnlock(t *testing.T) {
 	a := NewAccount()
-	err := a.AddBalance("USDT", 1000.0)
+	err := a.AddBalance("USDT", 1000)
 	assert.NoError(t, err)
 
 	balance, ok := a.Balance("USDT")
 	assert.True(t, ok)
-	assert.Equal(t, balance.Available, 1000.0)
+	assert.Equal(t, balance.Available, fixedpoint.Value(1000))
-	assert.Equal(t, balance.Locked, 0.0)
+	assert.Equal(t, balance.Locked, fixedpoint.Value(0))
 
-	err = a.LockBalance("USDT", 100.0)
+	err = a.LockBalance("USDT", fixedpoint.Value(100))
 	assert.NoError(t, err)
 
 	balance, ok = a.Balance("USDT")
 	assert.True(t, ok)
-	assert.Equal(t, balance.Available, 900.0)
+	assert.Equal(t, balance.Available, fixedpoint.Value(900))
-	assert.Equal(t, balance.Locked, 100.0)
+	assert.Equal(t, balance.Locked, fixedpoint.Value(100))
 
-	err = a.UnlockBalance("USDT", 100.0)
+	err = a.UnlockBalance("USDT", 100)
 	assert.NoError(t, err)
 	balance, ok = a.Balance("USDT")
 	assert.True(t, ok)
-	assert.Equal(t, balance.Available, 1000.0)
+	assert.Equal(t, balance.Available, fixedpoint.Value(1000))
-	assert.Equal(t, balance.Locked, 0.0)
+	assert.Equal(t, balance.Locked, fixedpoint.Value(0))
 }
 
 func TestAccountLockAndUse(t *testing.T) {
 	a := NewAccount()
-	err := a.AddBalance("USDT", 1000.0)
+	err := a.AddBalance("USDT", 1000)
 	assert.NoError(t, err)
 
 	balance, ok := a.Balance("USDT")
 	assert.True(t, ok)
-	assert.Equal(t, balance.Available, 1000.0)
+	assert.Equal(t, balance.Available, fixedpoint.Value(1000))
-	assert.Equal(t, balance.Locked, 0.0)
+	assert.Equal(t, balance.Locked, fixedpoint.Value(0))
 
-	err = a.LockBalance("USDT", 100.0)
+	err = a.LockBalance("USDT", 100)
 	assert.NoError(t, err)
 
 	balance, ok = a.Balance("USDT")
 	assert.True(t, ok)
-	assert.Equal(t, balance.Available, 900.0)
+	assert.Equal(t, balance.Available, fixedpoint.Value(900))
-	assert.Equal(t, balance.Locked, 100.0)
+	assert.Equal(t, balance.Locked, fixedpoint.Value(100))
 
-	err = a.UseLockedBalance("USDT", 100.0)
+	err = a.UseLockedBalance("USDT", 100)
 	assert.NoError(t, err)
 
-
 	balance, ok = a.Balance("USDT")
 	assert.True(t, ok)
-	assert.Equal(t, balance.Available, 900.0)
+	assert.Equal(t, balance.Available, fixedpoint.Value(900))
-	assert.Equal(t, balance.Locked, 0.0)
+	assert.Equal(t, balance.Locked, fixedpoint.Value(0))
 }

pkg/types/standardstream_callbacks.go

@@ -34,21 +34,21 @@ func (stream *StandardStream) EmitOrderUpdate(order Order) {
 	}
 }
 
-func (stream *StandardStream) OnBalanceSnapshot(cb func(balances map[string]Balance)) {
+func (stream *StandardStream) OnBalanceSnapshot(cb func(balances BalanceMap)) {
 	stream.balanceSnapshotCallbacks = append(stream.balanceSnapshotCallbacks, cb)
 }
 
-func (stream *StandardStream) EmitBalanceSnapshot(balances map[string]Balance) {
+func (stream *StandardStream) EmitBalanceSnapshot(balances BalanceMap) {
 	for _, cb := range stream.balanceSnapshotCallbacks {
 		cb(balances)
 	}
 }
 
-func (stream *StandardStream) OnBalanceUpdate(cb func(balances map[string]Balance)) {
+func (stream *StandardStream) OnBalanceUpdate(cb func(balances BalanceMap)) {
 	stream.balanceUpdateCallbacks = append(stream.balanceUpdateCallbacks, cb)
 }
 
-func (stream *StandardStream) EmitBalanceUpdate(balances map[string]Balance) {
+func (stream *StandardStream) EmitBalanceUpdate(balances BalanceMap) {
 	for _, cb := range stream.balanceUpdateCallbacks {
 		cb(balances)
 	}
@@ -101,9 +101,9 @@ type StandardStreamEventHub interface {
 
 	OnOrderUpdate(cb func(order Order))
 
-	OnBalanceSnapshot(cb func(balances map[string]Balance))
+	OnBalanceSnapshot(cb func(balances BalanceMap))
 
-	OnBalanceUpdate(cb func(balances map[string]Balance))
+	OnBalanceUpdate(cb func(balances BalanceMap))
 
 	OnKLineClosed(cb func(kline KLine))
 

pkg/types/stream.go

@@ -31,9 +31,9 @@ type StandardStream struct {
 	orderUpdateCallbacks []func(order Order)
 
 	// balance snapshot callbacks
-	balanceSnapshotCallbacks []func(balances map[string]Balance)
+	balanceSnapshotCallbacks []func(balances BalanceMap)
 
-	balanceUpdateCallbacks []func(balances map[string]Balance)
+	balanceUpdateCallbacks []func(balances BalanceMap)
 
 	kLineClosedCallbacks []func(kline KLine)