bbgo/pkg/backtest/matching.go

package backtest

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"git.qtrade.icu/lychiyu/bbgo/pkg/fixedpoint"
	"git.qtrade.icu/lychiyu/bbgo/pkg/types"
	"git.qtrade.icu/lychiyu/bbgo/pkg/util"
)

var orderID uint64 = 1
var tradeID uint64 = 1

func incOrderID() uint64 {
	return atomic.AddUint64(&orderID, 1)
}

func incTradeID() uint64 {
	return atomic.AddUint64(&tradeID, 1)
}

var klineMatchingLogger *logrus.Entry = nil

// FeeToken is used to simulate the exchange platform fee token
// This is to ease the back-testing environment for closing positions.
const FeeToken = "FEE"

var useFeeToken = true

func init() {
	logger := logrus.New()
	if v, ok := util.GetEnvVarBool("DEBUG_MATCHING"); ok && v {
		logger.SetLevel(logrus.DebugLevel)
	} else {
		logger.SetLevel(logrus.ErrorLevel)
	}
	klineMatchingLogger = logger.WithField("backtest", "klineEngine")

	if v, ok := util.GetEnvVarBool("BACKTEST_USE_FEE_TOKEN"); ok {
		useFeeToken = v
	}
}

// SimplePriceMatching implements a simple kline data driven matching engine for backtest
//
//go:generate callbackgen -type SimplePriceMatching
type SimplePriceMatching struct {
	Symbol string
	Market types.Market

	mu           sync.Mutex
	bidOrders    []types.Order
	askOrders    []types.Order
	closedOrders map[uint64]types.Order

	klineCache  map[types.Interval]types.KLine
	lastPrice   fixedpoint.Value
	lastKLine   types.KLine
	nextKLine   *types.KLine
	currentTime time.Time

	feeModeFunction FeeModeFunction

	account *types.Account

	tradeUpdateCallbacks   []func(trade types.Trade)
	orderUpdateCallbacks   []func(order types.Order)
	balanceUpdateCallbacks []func(balances types.BalanceMap)
}

func (m *SimplePriceMatching) CancelOrder(o types.Order) (types.Order, error) {
	found := false

	switch o.Side {

	case types.SideTypeBuy:
		m.mu.Lock()
		var orders []types.Order
		for _, order := range m.bidOrders {
			if o.OrderID == order.OrderID {
				found = true
				continue
			}
			orders = append(orders, order)
		}
		m.bidOrders = orders
		m.mu.Unlock()

	case types.SideTypeSell:
		m.mu.Lock()
		var orders []types.Order
		for _, order := range m.askOrders {
			if o.OrderID == order.OrderID {
				found = true
				continue
			}
			orders = append(orders, order)
		}
		m.askOrders = orders
		m.mu.Unlock()
	}

	if !found {
		return o, fmt.Errorf("cancel order failed, order %d not found: %+v", o.OrderID, o)
	}

	switch o.Side {
	case types.SideTypeBuy:
		if err := m.account.UnlockBalance(m.Market.QuoteCurrency, o.Price.Mul(o.Quantity)); err != nil {
			return o, err
		}

	case types.SideTypeSell:
		if err := m.account.UnlockBalance(m.Market.BaseCurrency, o.Quantity); err != nil {
			return o, err
		}
	}

	o.Status = types.OrderStatusCanceled
	m.EmitOrderUpdate(o)
	m.EmitBalanceUpdate(m.account.Balances())
	return o, nil
}

// PlaceOrder returns the created order object, executed trade (if any) and error
func (m *SimplePriceMatching) PlaceOrder(o types.SubmitOrder) (*types.Order, *types.Trade, error) {
	if o.Type == types.OrderTypeMarket {
		if m.lastPrice.IsZero() {
			panic("unexpected error: for market order, the last price can not be zero")
		}
	}

	isTaker := o.Type == types.OrderTypeMarket || isLimitTakerOrder(o, m.lastPrice)

	// price for checking account balance, default price
	price := o.Price

	switch o.Type {
	case types.OrderTypeMarket:
		price = m.Market.TruncatePrice(m.lastPrice)

	case types.OrderTypeStopMarket:
		// the actual price might be different.
		o.StopPrice = m.Market.TruncatePrice(o.StopPrice)
		price = o.StopPrice

	case types.OrderTypeLimit, types.OrderTypeStopLimit, types.OrderTypeLimitMaker:
		o.Price = m.Market.TruncatePrice(o.Price)
		price = o.Price
	}

	o.Quantity = m.Market.TruncateQuantity(o.Quantity)

	if o.Quantity.Compare(m.Market.MinQuantity) < 0 {
		return nil, nil, fmt.Errorf("order quantity %s is less than minQuantity %s, order: %+v", o.Quantity.String(), m.Market.MinQuantity.String(), o)
	}

	quoteQuantity := o.Quantity.Mul(price)
	if quoteQuantity.Compare(m.Market.MinNotional) < 0 {
		return nil, nil, fmt.Errorf("order amount %s is less than minNotional %s, order: %+v", quoteQuantity.String(), m.Market.MinNotional.String(), o)
	}

	switch o.Side {
	case types.SideTypeBuy:
		if err := m.account.LockBalance(m.Market.QuoteCurrency, quoteQuantity); err != nil {
			return nil, nil, err
		}

	case types.SideTypeSell:
		if err := m.account.LockBalance(m.Market.BaseCurrency, o.Quantity); err != nil {
			return nil, nil, err
		}
	}

	m.EmitBalanceUpdate(m.account.Balances())

	// start from one
	orderID := incOrderID()
	order := m.newOrder(o, orderID)

	if isTaker {
		var price fixedpoint.Value
		if order.Type == types.OrderTypeMarket {
			order.Price = m.Market.TruncatePrice(m.lastPrice)
			price = order.Price
		} else if order.Type == types.OrderTypeLimit {
			// if limit order's price is with the range of next kline
			// we assume it will be traded as a maker trade, and is traded at its original price
			// TODO: if it is treated as a maker trade, fee should be specially handled
			// otherwise, set NextKLine.Close(i.e., m.LastPrice) to be the taker traded price
			if m.nextKLine != nil && m.nextKLine.High.Compare(order.Price) > 0 && order.Side == types.SideTypeBuy {
				order.AveragePrice = order.Price
			} else if m.nextKLine != nil && m.nextKLine.Low.Compare(order.Price) < 0 && order.Side == types.SideTypeSell {
				order.AveragePrice = order.Price
			} else {
				order.AveragePrice = m.Market.TruncatePrice(m.lastPrice)
			}
			price = order.AveragePrice
		}

		// emit the order update for Status:New
		m.EmitOrderUpdate(order)

		// copy the order object to avoid side effect (for different callbacks)
		var order2 = order

		// emit trade before we publish order
		trade := m.newTradeFromOrder(&order2, false, price)
		m.executeTrade(trade)

		// unlock the rest balances for limit taker
		if order.Type == types.OrderTypeLimit {
			if order.AveragePrice.IsZero() {
				return nil, nil, fmt.Errorf("the average price of the given limit taker order can not be zero")
			}

			switch o.Side {
			case types.SideTypeBuy:
				// limit buy taker, the order price is higher than the current best ask price
				// the executed price is lower than the given price, so we will use less quote currency to buy the base asset.
				amount := order.Price.Sub(order.AveragePrice).Mul(order.Quantity)
				if amount.Sign() > 0 {
					if err := m.account.UnlockBalance(m.Market.QuoteCurrency, amount); err != nil {
						return nil, nil, err
					}
					m.EmitBalanceUpdate(m.account.Balances())
				}

			case types.SideTypeSell:
				// limit sell taker, the order price is lower than the current best bid price
				// the executed price is higher than the given price, so we will get more quote currency back
				amount := order.AveragePrice.Sub(order.Price).Mul(order.Quantity)
				if amount.Sign() > 0 {
					m.account.AddBalance(m.Market.QuoteCurrency, amount)
					m.EmitBalanceUpdate(m.account.Balances())
				}
			}
		}

		// update the order status
		order2.Status = types.OrderStatusFilled
		order2.ExecutedQuantity = order2.Quantity
		order2.IsWorking = false
		m.EmitOrderUpdate(order2)

		// let the exchange emit the "FILLED" order update (we need the closed order)
		// m.EmitOrderUpdate(order2)
		return &order2, &trade, nil
	}

	// For limit maker orders (open status)
	switch o.Side {

	case types.SideTypeBuy:
		m.mu.Lock()
		m.bidOrders = append(m.bidOrders, order)
		m.mu.Unlock()

	case types.SideTypeSell:
		m.mu.Lock()
		m.askOrders = append(m.askOrders, order)
		m.mu.Unlock()
	}

	m.EmitOrderUpdate(order) // emit order New status
	return &order, nil, nil
}

func (m *SimplePriceMatching) executeTrade(trade types.Trade) {
	var err error
	// execute trade, update account balances
	if trade.IsBuyer {
		err = m.account.UseLockedBalance(m.Market.QuoteCurrency, trade.QuoteQuantity)

		// all-in buy trade, we can only deduct the fee from the quote quantity and re-calculate the base quantity
		switch trade.FeeCurrency {
		case m.Market.QuoteCurrency:
			m.account.AddBalance(m.Market.QuoteCurrency, trade.Fee.Neg())
			m.account.AddBalance(m.Market.BaseCurrency, trade.Quantity)
		case m.Market.BaseCurrency:
			m.account.AddBalance(m.Market.BaseCurrency, trade.Quantity.Sub(trade.Fee))
		default:
			m.account.AddBalance(m.Market.BaseCurrency, trade.Quantity)
		}

	} else { // sell trade
		err = m.account.UseLockedBalance(m.Market.BaseCurrency, trade.Quantity)

		switch trade.FeeCurrency {
		case m.Market.QuoteCurrency:
			m.account.AddBalance(m.Market.QuoteCurrency, trade.QuoteQuantity.Sub(trade.Fee))
		case m.Market.BaseCurrency:
			m.account.AddBalance(m.Market.BaseCurrency, trade.Fee.Neg())
			m.account.AddBalance(m.Market.QuoteCurrency, trade.QuoteQuantity)
		default:
			m.account.AddBalance(m.Market.QuoteCurrency, trade.QuoteQuantity)
		}
	}

	if err != nil {
		panic(errors.Wrapf(err, "executeTrade exception, wanted to use more than the locked balance"))
	}

	m.EmitTradeUpdate(trade)
	m.EmitBalanceUpdate(m.account.Balances())
}

func (m *SimplePriceMatching) getFeeRate(isMaker bool) (feeRate fixedpoint.Value) {
	// BINANCE uses 0.1% for both maker and taker
	// MAX uses 0.050% for maker and 0.15% for taker
	if isMaker {
		feeRate = m.account.MakerFeeRate
	} else {
		feeRate = m.account.TakerFeeRate
	}
	return feeRate
}

func (m *SimplePriceMatching) newTradeFromOrder(order *types.Order, isMaker bool, price fixedpoint.Value) types.Trade {
	// BINANCE uses 0.1% for both maker and taker
	// MAX uses 0.050% for maker and 0.15% for taker
	var feeRate = m.getFeeRate(isMaker)
	var quoteQuantity = order.Quantity.Mul(price)
	var fee fixedpoint.Value
	var feeCurrency string

	if m.feeModeFunction != nil {
		fee, feeCurrency = m.feeModeFunction(order, &m.Market, feeRate)
	} else {
		fee, feeCurrency = feeModeFunctionQuote(order, &m.Market, feeRate)
	}

	// update order time
	order.UpdateTime = types.Time(m.currentTime)

	var id = incTradeID()
	return types.Trade{
		ID:            id,
		OrderID:       order.OrderID,
		Exchange:      types.ExchangeBacktest,
		Price:         price,
		Quantity:      order.Quantity,
		QuoteQuantity: quoteQuantity,
		Symbol:        order.Symbol,
		Side:          order.Side,
		IsBuyer:       order.Side == types.SideTypeBuy,
		IsMaker:       isMaker,
		Time:          types.Time(m.currentTime),
		Fee:           fee,
		FeeCurrency:   feeCurrency,
	}
}

// buyToPrice means price go up and the limit sell should be triggered
func (m *SimplePriceMatching) buyToPrice(price fixedpoint.Value) (closedOrders []types.Order, trades []types.Trade) {
	klineMatchingLogger.Debugf("kline buy to price %s", price.String())

	var bidOrders []types.Order
	for _, o := range m.bidOrders {
		switch o.Type {

		case types.OrderTypeStopMarket:
			// the price is still lower than the stop price, we will put the order back to the list
			if price.Compare(o.StopPrice) < 0 {
				// not triggering it, put it back
				bidOrders = append(bidOrders, o)
				break
			}

			o.Type = types.OrderTypeMarket
			o.ExecutedQuantity = o.Quantity
			o.Price = price
			o.Status = types.OrderStatusFilled
			closedOrders = append(closedOrders, o)

		case types.OrderTypeStopLimit:
			// the price is still lower than the stop price, we will put the order back to the list
			if price.Compare(o.StopPrice) < 0 {
				bidOrders = append(bidOrders, o)
				break
			}

			// convert this order to limit order
			// we use value object here, so it's a copy
			o.Type = types.OrderTypeLimit

			// is it a taker order?
			// higher than the current price, then it's a taker order
			if o.Price.Compare(price) >= 0 {
				// limit buy taker order, move it to the closed order
				// we assume that we have no price slippage here, so the latest price will be the executed price
				o.AveragePrice = price
				o.ExecutedQuantity = o.Quantity
				o.Status = types.OrderStatusFilled
				closedOrders = append(closedOrders, o)
			} else {
				// keep it as a maker order
				bidOrders = append(bidOrders, o)
			}
		default:
			bidOrders = append(bidOrders, o)
		}
	}
	m.bidOrders = bidOrders

	var askOrders []types.Order
	for _, o := range m.askOrders {
		switch o.Type {

		case types.OrderTypeStopMarket:
			// should we trigger the order
			if price.Compare(o.StopPrice) < 0 {
				// not triggering it, put it back
				askOrders = append(askOrders, o)
				break
			}

			o.Type = types.OrderTypeMarket
			o.ExecutedQuantity = o.Quantity
			o.Price = price
			o.Status = types.OrderStatusFilled
			closedOrders = append(closedOrders, o)

		case types.OrderTypeStopLimit:
			// should we trigger the order?
			if price.Compare(o.StopPrice) < 0 {
				askOrders = append(askOrders, o)
				break
			}

			o.Type = types.OrderTypeLimit

			// is it a taker order?
			// higher than the current price, then it's a taker order
			if o.Price.Compare(price) <= 0 {
				// limit sell order as taker, move it to the closed order
				// we assume that we have no price slippage here, so the latest price will be the executed price
				// TODO: simulate slippage here
				o.AveragePrice = price
				o.ExecutedQuantity = o.Quantity
				o.Status = types.OrderStatusFilled
				closedOrders = append(closedOrders, o)
			} else {
				// maker order
				askOrders = append(askOrders, o)
			}

		case types.OrderTypeLimit, types.OrderTypeLimitMaker:
			if price.Compare(o.Price) >= 0 {
				o.ExecutedQuantity = o.Quantity
				o.Status = types.OrderStatusFilled
				closedOrders = append(closedOrders, o)
			} else {
				askOrders = append(askOrders, o)
			}

		default:
			askOrders = append(askOrders, o)
		}

	}

	m.askOrders = askOrders
	m.lastPrice = price

	for i := range closedOrders {
		o := closedOrders[i]
		executedPrice := o.Price
		if !o.AveragePrice.IsZero() {
			executedPrice = o.AveragePrice
		}

		trade := m.newTradeFromOrder(&o, !isTakerOrder(o), executedPrice)
		m.executeTrade(trade)
		closedOrders[i] = o

		trades = append(trades, trade)

		m.EmitOrderUpdate(o)

		m.closedOrders[o.OrderID] = o
	}

	return closedOrders, trades
}

// sellToPrice simulates the price trend in down direction.
// When price goes down, buy orders should be executed, and the stop orders should be triggered.
func (m *SimplePriceMatching) sellToPrice(price fixedpoint.Value) (closedOrders []types.Order, trades []types.Trade) {
	klineMatchingLogger.Debugf("kline sell to price %s", price.String())

	// in this section we handle --- the price goes lower, and we trigger the stop sell
	var askOrders []types.Order
	for _, o := range m.askOrders {
		switch o.Type {

		case types.OrderTypeStopMarket:
			// should we trigger the order
			if price.Compare(o.StopPrice) > 0 {
				askOrders = append(askOrders, o)
				break
			}

			o.Type = types.OrderTypeMarket
			o.ExecutedQuantity = o.Quantity
			o.Price = price
			o.Status = types.OrderStatusFilled
			closedOrders = append(closedOrders, o)

		case types.OrderTypeStopLimit:
			// if the price is lower than the stop price
			// we should trigger the stop sell order
			if price.Compare(o.StopPrice) > 0 {
				askOrders = append(askOrders, o)
				break
			}

			o.Type = types.OrderTypeLimit

			// handle TAKER SELL
			// if the order price is lower than the current price
			// it's a taker order
			if o.Price.Compare(price) <= 0 {
				o.AveragePrice = price
				o.ExecutedQuantity = o.Quantity
				o.Status = types.OrderStatusFilled
				closedOrders = append(closedOrders, o)
			} else {
				askOrders = append(askOrders, o)
			}

		default:
			askOrders = append(askOrders, o)
		}
	}
	m.askOrders = askOrders

	var bidOrders []types.Order
	for _, o := range m.bidOrders {
		switch o.Type {

		case types.OrderTypeStopMarket:
			// price goes down and if the stop price is still lower than the current price
			// or the stop price is not touched
			// then we should skip this order
			if price.Compare(o.StopPrice) > 0 {
				bidOrders = append(bidOrders, o)
				break
			}

			o.Type = types.OrderTypeMarket
			o.ExecutedQuantity = o.Quantity
			o.Price = price
			o.Status = types.OrderStatusFilled
			closedOrders = append(closedOrders, o)

		case types.OrderTypeStopLimit:
			// price goes down and if the stop price is still lower than the current price
			// or the stop price is not touched
			// then we should skip this order
			if price.Compare(o.StopPrice) > 0 {
				bidOrders = append(bidOrders, o)
				break
			}

			o.Type = types.OrderTypeLimit

			// handle TAKER order
			if o.Price.Compare(price) >= 0 {
				o.AveragePrice = price
				o.ExecutedQuantity = o.Quantity
				o.Status = types.OrderStatusFilled
				closedOrders = append(closedOrders, o)
			} else {
				bidOrders = append(bidOrders, o)
			}

		case types.OrderTypeLimit, types.OrderTypeLimitMaker:
			if price.Compare(o.Price) <= 0 {
				o.ExecutedQuantity = o.Quantity
				o.Status = types.OrderStatusFilled
				closedOrders = append(closedOrders, o)
			} else {
				bidOrders = append(bidOrders, o)
			}

		default:
			bidOrders = append(bidOrders, o)
		}
	}

	m.bidOrders = bidOrders
	m.lastPrice = price

	for i := range closedOrders {
		o := closedOrders[i]
		executedPrice := o.Price
		if !o.AveragePrice.IsZero() {
			executedPrice = o.AveragePrice
		}

		trade := m.newTradeFromOrder(&o, !isTakerOrder(o), executedPrice)
		m.executeTrade(trade)
		closedOrders[i] = o

		trades = append(trades, trade)

		m.EmitOrderUpdate(o)

		m.closedOrders[o.OrderID] = o
	}

	return closedOrders, trades
}

func (m *SimplePriceMatching) getOrder(orderID uint64) (types.Order, bool) {
	if o, ok := m.closedOrders[orderID]; ok {
		return o, true
	}

	for _, o := range m.bidOrders {
		if o.OrderID == orderID {
			return o, true
		}
	}

	for _, o := range m.askOrders {
		if o.OrderID == orderID {
			return o, true
		}
	}

	return types.Order{}, false
}

func (m *SimplePriceMatching) processKLine(kline types.KLine) {
	m.currentTime = kline.EndTime.Time()

	if m.lastPrice.IsZero() {
		m.lastPrice = kline.Open
	} else {
		if m.lastPrice.Compare(kline.Open) > 0 {
			m.sellToPrice(kline.Open)
		} else {
			m.buyToPrice(kline.Open)
		}
	}

	switch kline.Direction() {
	case types.DirectionDown:
		if kline.High.Compare(kline.Open) >= 0 {
			m.buyToPrice(kline.High)
		}

		// if low is lower than close, sell to low first, and then buy up to close
		if kline.Low.Compare(kline.Close) < 0 {
			m.sellToPrice(kline.Low)
			m.buyToPrice(kline.Close)
		} else {
			m.sellToPrice(kline.Close)
		}

	case types.DirectionUp:
		if kline.Low.Compare(kline.Open) <= 0 {
			m.sellToPrice(kline.Low)
		}

		if kline.High.Compare(kline.Close) > 0 {
			m.buyToPrice(kline.High)
			m.sellToPrice(kline.Close)
		} else {
			m.buyToPrice(kline.Close)
		}
	default: // no trade up or down
		if m.lastPrice.IsZero() {
			m.buyToPrice(kline.Close)
		}
	}

	m.lastKLine = kline
}

func (m *SimplePriceMatching) newOrder(o types.SubmitOrder, orderID uint64) types.Order {
	return types.Order{
		OrderID:          orderID,
		SubmitOrder:      o,
		Exchange:         types.ExchangeBacktest,
		Status:           types.OrderStatusNew,
		ExecutedQuantity: fixedpoint.Zero,
		IsWorking:        true,
		CreationTime:     types.Time(m.currentTime),
		UpdateTime:       types.Time(m.currentTime),
	}
}

func isTakerOrder(o types.Order) bool {
	if o.AveragePrice.IsZero() {
		return false
	}

	switch o.Side {
	case types.SideTypeBuy:
		return o.AveragePrice.Compare(o.Price) < 0

	case types.SideTypeSell:
		return o.AveragePrice.Compare(o.Price) > 0

	}
	return false
}

func isLimitTakerOrder(o types.SubmitOrder, currentPrice fixedpoint.Value) bool {
	if currentPrice.IsZero() {
		return false
	}

	return o.Type == types.OrderTypeLimit && ((o.Side == types.SideTypeBuy && o.Price.Compare(currentPrice) >= 0) ||
		(o.Side == types.SideTypeSell && o.Price.Compare(currentPrice) <= 0))
}