bbgo_origin/pkg/datasource/csvsource/csv_tick_decoder.go

package csvsource

import (
	"encoding/csv"
	"strconv"
	"time"

	"github.com/c9s/bbgo/pkg/fixedpoint"
	"github.com/c9s/bbgo/pkg/types"
)

// CSVTickDecoder is an extension point for CSVTickReader to support custom file formats.
type CSVTickDecoder func(record []string, index int) (*CsvTick, error)

// NewBinanceCSVTickReader creates a new CSVTickReader for Binance CSV files.
func NewBinanceCSVTickReader(csv *csv.Reader) *CSVTickReader {
	return &CSVTickReader{
		csv:     csv,
		decoder: BinanceCSVTickDecoder,
	}
}

// BinanceCSVKLineDecoder decodes a CSV record from Binance into a CsvTick.
func BinanceCSVTickDecoder(row []string, _ int) (*CsvTick, error) {
	if len(row) < 5 {
		return nil, ErrNotEnoughColumns
	}
	// example csv row for some reason some properties are duplicated in their csv
	// id, price, qty, base_qty, base_qty, time, is_buyer_maker, is_buyer_maker,
	// 11782578,6.00000000,1.00000000,14974844,14974844,1698623884463,True
	id, err := strconv.ParseUint(row[0], 10, 64)
	if err != nil {
		return nil, ErrInvalidIDFormat
	}
	price, err := fixedpoint.NewFromString(row[1])
	if err != nil {
		return nil, ErrInvalidPriceFormat
	}
	qty, err := fixedpoint.NewFromString(row[2])
	if err != nil {
		return nil, ErrInvalidVolumeFormat
	}
	baseQty, err := fixedpoint.NewFromString(row[3])
	if err != nil {
		return nil, ErrInvalidVolumeFormat
	}
	isBuyerMaker, err := strconv.ParseBool(row[6])
	if err != nil {
		return nil, err
	}
	// isBuyerMaker=false trade will qualify as BUY.
	side := types.SideTypeBuy
	if isBuyerMaker {
		side = types.SideTypeSell
	}
	n, err := strconv.ParseFloat(row[5], 64)
	if err != nil {
		return nil, ErrInvalidTimeFormat
	}
	ts := time.Unix(int64(n), 0)
	return &CsvTick{
		TradeID:         id,
		Exchange:        types.ExchangeBinance,
		Side:            side,
		Size:            qty,
		Price:           price,
		IsBuyerMaker:    isBuyerMaker,
		HomeNotional:    price.Mul(qty),
		ForeignNotional: price.Mul(baseQty),
		Timestamp:       types.NewMillisecondTimestampFromInt(ts.UnixMilli()),
		// Symbol: must be overwritten - info not in csv,
		// TickDirection: would need to keep last tick in memory to compare tick direction,
	}, nil
}

// NewBinanceCSVTickReader creates a new CSVTickReader for Bybit CSV files.
func NewBybitCSVTickReader(csv *csv.Reader) *CSVTickReader {
	return &CSVTickReader{
		csv:     csv,
		decoder: BybitCSVTickDecoder,
	}
}

// BybitCSVTickDecoder decodes a CSV record from Bybit into a CsvTick.
func BybitCSVTickDecoder(row []string, index int) (*CsvTick, error) {
	// example csv row
	// timestamp,symbol,side,size,price,tickDirection,trdMatchID,grossValue,homeNotional,foreignNotional
	// 1649054912,FXSUSDT,Buy,0.01,38.32,PlusTick,9c30abaf-80ae-5ebf-9850-58fe7ed4bac8,3.832e+07,0.01,0.3832
	if len(row) < 9 {
		return nil, ErrNotEnoughColumns
	}
	if index == 0 {
		return nil, nil
	}
	side, err := types.StrToSideType(row[2])
	if err != nil {
		return nil, ErrInvalidOrderSideFormat
	}
	size, err := fixedpoint.NewFromString(row[3])
	if err != nil {
		return nil, ErrInvalidVolumeFormat
	}
	price, err := fixedpoint.NewFromString(row[4])
	if err != nil {
		return nil, ErrInvalidPriceFormat
	}
	hn, err := fixedpoint.NewFromString(row[8])
	if err != nil {
		return nil, ErrInvalidVolumeFormat
	}
	fn, err := fixedpoint.NewFromString(row[9])
	if err != nil {
		return nil, ErrInvalidVolumeFormat
	}
	n, err := strconv.ParseFloat(row[0], 64) // startTime eg 1696982287.4922
	if err != nil {
		return nil, ErrInvalidTimeFormat
	}
	ts := time.Unix(int64(n), 0)
	return &CsvTick{
		TradeID:         uint64(index),
		Symbol:          row[1],
		Exchange:        types.ExchangeBybit,
		Side:            side,
		Size:            size,
		Price:           price,
		HomeNotional:    hn,
		ForeignNotional: fn,
		TickDirection:   row[5], // todo does this seem promising to define for other exchanges too?
		Timestamp:       types.NewMillisecondTimestampFromInt(ts.UnixMilli()),
	}, nil
}

// NewOKExCSVTickReader creates a new CSVTickReader for OKEx CSV files.
func NewOKExCSVTickReader(csv *csv.Reader) *CSVTickReader {
	return &CSVTickReader{
		csv:     csv,
		decoder: OKExCSVTickDecoder,
	}
}

// OKExCSVKLineDecoder decodes a CSV record from OKEx into a CsvTick.
func OKExCSVTickDecoder(row []string, index int) (*CsvTick, error) {
	if len(row) < 5 {
		return nil, ErrNotEnoughColumns
	}
	if index == 0 {
		return nil, nil
	}
	// example csv row for OKeX
	// trade_id, side,	size,	price,	created_time
	// 134642,	sell,	6.2638	6.507	1.69975E+12
	id, err := strconv.ParseInt(row[0], 10, 64)
	if err != nil {
		return nil, ErrInvalidIDFormat
	}
	price, err := fixedpoint.NewFromString(row[3])
	if err != nil {
		return nil, ErrInvalidPriceFormat
	}

	qty, err := fixedpoint.NewFromString(row[2])
	if err != nil {
		return nil, ErrInvalidVolumeFormat
	}

	side := types.SideTypeBuy
	isBuyerMaker := false
	if row[1] == "sell" {
		side = types.SideTypeSell
		isBuyerMaker = true
	}

	n, err := strconv.ParseFloat(row[4], 64) // startTime
	if err != nil {
		return nil, ErrInvalidTimeFormat
	}

	ts := time.UnixMilli(int64(n))
	return &CsvTick{
		TradeID:      uint64(id),
		Exchange:     types.ExchangeOKEx,
		Side:         side,
		Size:         qty,
		Price:        price,
		IsBuyerMaker: isBuyerMaker,
		HomeNotional: price.Mul(qty),
		Timestamp:    types.NewMillisecondTimestampFromInt(ts.UnixMilli()),
		// ForeignNotional: // info not in csv
		// Symbol: must be overwritten - info not in csv
		// TickDirection: would need to keep last tick in memory to compare tick direction,
	}, nil
}