mirror of
https://github.com/c9s/bbgo.git
synced 2024-11-10 09:11:55 +00:00
release candidate csvsource backtest
This commit is contained in:
parent
e23e8fde1d
commit
af1e63f345
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -14,6 +14,7 @@
|
||||||
*.out
|
*.out
|
||||||
|
|
||||||
.idea
|
.idea
|
||||||
|
.vscode
|
||||||
|
|
||||||
# Dependency directories (remove the comment below to include it)
|
# Dependency directories (remove the comment below to include it)
|
||||||
# vendor/
|
# vendor/
|
||||||
|
@ -48,6 +49,7 @@ testoutput
|
||||||
|
|
||||||
*.swp
|
*.swp
|
||||||
/pkg/backtest/assets.go
|
/pkg/backtest/assets.go
|
||||||
|
/data/backtest
|
||||||
|
|
||||||
coverage.txt
|
coverage.txt
|
||||||
coverage_dum.txt
|
coverage_dum.txt
|
||||||
|
|
|
@ -1,9 +1,13 @@
|
||||||
## Back-testing
|
## Back-testing
|
||||||
|
|
||||||
*Before you start back-testing, you need to setup [MySQL](../../README.md#configure-mysql-database) or [SQLite3
|
Currently bbgo supports two ways to run backtests:
|
||||||
|
|
||||||
|
1: Through csv data source (supported right now are binance, bybit and OkEx)
|
||||||
|
|
||||||
|
2: Alternatively run backtests through [MySQL](../../README.md#configure-mysql-database) or [SQLite3
|
||||||
](../../README.md#configure-sqlite3-database). Using MySQL is highly recommended.*
|
](../../README.md#configure-sqlite3-database). Using MySQL is highly recommended.*
|
||||||
|
|
||||||
First, you need to add the back-testing config to your `bbgo.yaml`:
|
Let's start by adding the back-testing section to your config eg: `bbgo.yaml`:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
backtest:
|
backtest:
|
||||||
|
@ -41,8 +45,11 @@ Note on date formats, the following date formats are supported:
|
||||||
* RFC822, which looks like `02 Jan 06 15:04 MST`
|
* RFC822, which looks like `02 Jan 06 15:04 MST`
|
||||||
* You can also use `2021-11-26T15:04:56`
|
* You can also use `2021-11-26T15:04:56`
|
||||||
|
|
||||||
And then, you can sync remote exchange k-lines (candle bars) data for back-testing:
|
And then, you can sync remote exchange k-lines (candle bars) data for back-testing through csv data source:
|
||||||
|
```sh
|
||||||
|
bbgo backtest -v --csv --verify --config config/grid.yaml
|
||||||
|
```
|
||||||
|
or use the sql data source like so:
|
||||||
```sh
|
```sh
|
||||||
bbgo backtest -v --sync --config config/grid.yaml
|
bbgo backtest -v --sync --config config/grid.yaml
|
||||||
```
|
```
|
||||||
|
@ -67,6 +74,11 @@ Run back-test:
|
||||||
```sh
|
```sh
|
||||||
bbgo backtest --base-asset-baseline --config config/grid.yaml
|
bbgo backtest --base-asset-baseline --config config/grid.yaml
|
||||||
```
|
```
|
||||||
|
or through csv data source
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bbgo backtest -v --csv --base-asset-baseline --config config/grid.yaml --output data/backtest
|
||||||
|
```
|
||||||
|
|
||||||
If you're developing a strategy, you might want to start with a command like this:
|
If you're developing a strategy, you might want to start with a command like this:
|
||||||
|
|
||||||
|
|
|
@ -55,7 +55,7 @@ var ErrEmptyOrderType = errors.New("order type can not be empty string")
|
||||||
type Exchange struct {
|
type Exchange struct {
|
||||||
sourceName types.ExchangeName
|
sourceName types.ExchangeName
|
||||||
publicExchange types.Exchange
|
publicExchange types.Exchange
|
||||||
srv *service.BacktestService
|
srv service.BackTestable
|
||||||
currentTime time.Time
|
currentTime time.Time
|
||||||
|
|
||||||
account *types.Account
|
account *types.Account
|
||||||
|
@ -78,7 +78,7 @@ type Exchange struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewExchange(
|
func NewExchange(
|
||||||
sourceName types.ExchangeName, sourceExchange types.Exchange, srv *service.BacktestService, config *bbgo.Backtest,
|
sourceName types.ExchangeName, sourceExchange types.Exchange, srv service.BackTestable, config *bbgo.Backtest,
|
||||||
) (*Exchange, error) {
|
) (*Exchange, error) {
|
||||||
ex := sourceExchange
|
ex := sourceExchange
|
||||||
|
|
||||||
|
@ -366,6 +366,7 @@ func (e *Exchange) SubscribeMarketData(
|
||||||
loadedIntervals[sub.Options.Interval] = struct{}{}
|
loadedIntervals[sub.Options.Interval] = struct{}{}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
// todo support stream back test with csv tick source
|
||||||
// Since Environment is not yet been injected at this point, no hard error
|
// Since Environment is not yet been injected at this point, no hard error
|
||||||
log.Errorf("stream channel %s is not supported in backtest", sub.Channel)
|
log.Errorf("stream channel %s is not supported in backtest", sub.Channel)
|
||||||
}
|
}
|
||||||
|
@ -394,6 +395,7 @@ func (e *Exchange) SubscribeMarketData(
|
||||||
log.Infof("querying klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals)
|
log.Infof("querying klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Infof("querying klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals)
|
||||||
if len(symbols) == 0 {
|
if len(symbols) == 0 {
|
||||||
log.Warnf("empty symbols, will not query kline data from the database")
|
log.Warnf("empty symbols, will not query kline data from the database")
|
||||||
|
|
||||||
|
|
|
@ -68,49 +68,21 @@ func ReadSummaryReport(filename string) (*SummaryReport, error) {
|
||||||
// SessionSymbolReport is the report per exchange session
|
// SessionSymbolReport is the report per exchange session
|
||||||
// trades are merged, collected and re-calculated
|
// trades are merged, collected and re-calculated
|
||||||
type SessionSymbolReport struct {
|
type SessionSymbolReport struct {
|
||||||
Exchange types.ExchangeName `json:"exchange"`
|
Exchange types.ExchangeName `json:"exchange"`
|
||||||
Symbol string `json:"symbol,omitempty"`
|
Symbol string `json:"symbol,omitempty"`
|
||||||
Intervals []types.Interval `json:"intervals,omitempty"`
|
Intervals []types.Interval `json:"intervals,omitempty"`
|
||||||
Subscriptions []types.Subscription `json:"subscriptions"`
|
Subscriptions []types.Subscription `json:"subscriptions"`
|
||||||
Market types.Market `json:"market"`
|
Market types.Market `json:"market"`
|
||||||
LastPrice fixedpoint.Value `json:"lastPrice,omitempty"`
|
LastPrice fixedpoint.Value `json:"lastPrice,omitempty"`
|
||||||
StartPrice fixedpoint.Value `json:"startPrice,omitempty"`
|
StartPrice fixedpoint.Value `json:"startPrice,omitempty"`
|
||||||
PnL *pnl.AverageCostPnLReport `json:"pnl,omitempty"`
|
PnL *pnl.AverageCostPnLReport `json:"pnl,omitempty"`
|
||||||
InitialBalances types.BalanceMap `json:"initialBalances,omitempty"`
|
InitialBalances types.BalanceMap `json:"initialBalances,omitempty"`
|
||||||
FinalBalances types.BalanceMap `json:"finalBalances,omitempty"`
|
FinalBalances types.BalanceMap `json:"finalBalances,omitempty"`
|
||||||
Manifests Manifests `json:"manifests,omitempty"`
|
Manifests Manifests `json:"manifests,omitempty"`
|
||||||
TradeCount fixedpoint.Value `json:"tradeCount,omitempty"`
|
Sharpe fixedpoint.Value `json:"sharpeRatio"`
|
||||||
RoundTurnCount fixedpoint.Value `json:"roundTurnCount,omitempty"`
|
Sortino fixedpoint.Value `json:"sortinoRatio"`
|
||||||
TotalNetProfit fixedpoint.Value `json:"totalNetProfit,omitempty"`
|
ProfitFactor fixedpoint.Value `json:"profitFactor"`
|
||||||
AvgNetProfit fixedpoint.Value `json:"avgNetProfit,omitempty"`
|
WinningRatio fixedpoint.Value `json:"winningRatio"`
|
||||||
GrossProfit fixedpoint.Value `json:"grossProfit,omitempty"`
|
|
||||||
GrossLoss fixedpoint.Value `json:"grossLoss,omitempty"`
|
|
||||||
PRR fixedpoint.Value `json:"prr,omitempty"`
|
|
||||||
PercentProfitable fixedpoint.Value `json:"percentProfitable,omitempty"`
|
|
||||||
MaxDrawdown fixedpoint.Value `json:"maxDrawdown,omitempty"`
|
|
||||||
AverageDrawdown fixedpoint.Value `json:"avgDrawdown,omitempty"`
|
|
||||||
MaxProfit fixedpoint.Value `json:"maxProfit,omitempty"`
|
|
||||||
MaxLoss fixedpoint.Value `json:"maxLoss,omitempty"`
|
|
||||||
AvgProfit fixedpoint.Value `json:"avgProfit,omitempty"`
|
|
||||||
AvgLoss fixedpoint.Value `json:"avgLoss,omitempty"`
|
|
||||||
TotalTimeInMarketSec int64 `json:"totalTimeInMarketSec,omitempty"`
|
|
||||||
AvgHoldSec int64 `json:"avgHoldSec,omitempty"`
|
|
||||||
WinningCount int `json:"winningCount,omitempty"`
|
|
||||||
LosingCount int `json:"losingCount,omitempty"`
|
|
||||||
MaxLossStreak int `json:"maxLossStreak,omitempty"`
|
|
||||||
Sharpe fixedpoint.Value `json:"sharpeRatio"`
|
|
||||||
AnnualHistoricVolatility fixedpoint.Value `json:"annualHistoricVolatility,omitempty"`
|
|
||||||
CAGR fixedpoint.Value `json:"cagr,omitempty"`
|
|
||||||
Calmar fixedpoint.Value `json:"calmar,omitempty"`
|
|
||||||
Sterling fixedpoint.Value `json:"sterling,omitempty"`
|
|
||||||
Burke fixedpoint.Value `json:"burke,omitempty"`
|
|
||||||
Kelly fixedpoint.Value `json:"kelly,omitempty"`
|
|
||||||
OptimalF fixedpoint.Value `json:"optimalF,omitempty"`
|
|
||||||
StatN fixedpoint.Value `json:"statN,omitempty"`
|
|
||||||
StdErr fixedpoint.Value `json:"statNStdErr,omitempty"`
|
|
||||||
Sortino fixedpoint.Value `json:"sortinoRatio"`
|
|
||||||
ProfitFactor fixedpoint.Value `json:"profitFactor"`
|
|
||||||
WinningRatio fixedpoint.Value `json:"winningRatio"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *SessionSymbolReport) InitialEquityValue() fixedpoint.Value {
|
func (r *SessionSymbolReport) InitialEquityValue() fixedpoint.Value {
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/datasource/csvsource"
|
||||||
"github.com/c9s/bbgo/pkg/datatype"
|
"github.com/c9s/bbgo/pkg/datatype"
|
||||||
"github.com/c9s/bbgo/pkg/dynamic"
|
"github.com/c9s/bbgo/pkg/dynamic"
|
||||||
"github.com/c9s/bbgo/pkg/fixedpoint"
|
"github.com/c9s/bbgo/pkg/fixedpoint"
|
||||||
|
@ -150,7 +151,8 @@ type Backtest struct {
|
||||||
Sessions []string `json:"sessions" yaml:"sessions"`
|
Sessions []string `json:"sessions" yaml:"sessions"`
|
||||||
|
|
||||||
// sync 1 second interval KLines
|
// sync 1 second interval KLines
|
||||||
SyncSecKLines bool `json:"syncSecKLines,omitempty" yaml:"syncSecKLines,omitempty"`
|
SyncSecKLines bool `json:"syncSecKLines,omitempty" yaml:"syncSecKLines,omitempty"`
|
||||||
|
CsvSource *csvsource.CsvConfig `json:"csvConfig,omitempty" yaml:"csvConfig,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Backtest) GetAccount(n string) BacktestAccount {
|
func (b *Backtest) GetAccount(n string) BacktestAccount {
|
||||||
|
|
|
@ -42,9 +42,9 @@ var defaultSyncBufferPeriod = 30 * time.Minute
|
||||||
// IsBackTesting is a global variable that indicates the current environment is back-test or not.
|
// IsBackTesting is a global variable that indicates the current environment is back-test or not.
|
||||||
var IsBackTesting = false
|
var IsBackTesting = false
|
||||||
|
|
||||||
var BackTestService *service.BacktestService
|
var BackTestService service.BackTestable
|
||||||
|
|
||||||
func SetBackTesting(s *service.BacktestService) {
|
func SetBackTesting(s service.BackTestable) {
|
||||||
BackTestService = s
|
BackTestService = s
|
||||||
IsBackTesting = s != nil
|
IsBackTesting = s != nil
|
||||||
}
|
}
|
||||||
|
@ -87,7 +87,7 @@ type Environment struct {
|
||||||
TradeService *service.TradeService
|
TradeService *service.TradeService
|
||||||
ProfitService *service.ProfitService
|
ProfitService *service.ProfitService
|
||||||
PositionService *service.PositionService
|
PositionService *service.PositionService
|
||||||
BacktestService *service.BacktestService
|
BacktestService service.BackTestable
|
||||||
RewardService *service.RewardService
|
RewardService *service.RewardService
|
||||||
MarginService *service.MarginService
|
MarginService *service.MarginService
|
||||||
SyncService *service.SyncService
|
SyncService *service.SyncService
|
||||||
|
|
|
@ -31,6 +31,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
BacktestCmd.Flags().Bool("csv", false, "use csv data source for exchange (if supported)")
|
||||||
BacktestCmd.Flags().Bool("sync", false, "sync backtest data")
|
BacktestCmd.Flags().Bool("sync", false, "sync backtest data")
|
||||||
BacktestCmd.Flags().Bool("sync-only", false, "sync backtest data only, do not run backtest")
|
BacktestCmd.Flags().Bool("sync-only", false, "sync backtest data only, do not run backtest")
|
||||||
BacktestCmd.Flags().String("sync-from", "", "sync backtest data from the given time, which will override the time range in the backtest config")
|
BacktestCmd.Flags().String("sync-from", "", "sync backtest data from the given time, which will override the time range in the backtest config")
|
||||||
|
@ -75,6 +76,11 @@ var BacktestCmd = &cobra.Command{
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
modeCsv, err := cmd.Flags().GetBool("csv")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
wantSync, err := cmd.Flags().GetBool("sync")
|
wantSync, err := cmd.Flags().GetBool("sync")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -154,15 +160,29 @@ var BacktestCmd = &cobra.Command{
|
||||||
log.Infof("starting backtest with startTime %s", startTime.Format(time.RFC3339))
|
log.Infof("starting backtest with startTime %s", startTime.Format(time.RFC3339))
|
||||||
|
|
||||||
environ := bbgo.NewEnvironment()
|
environ := bbgo.NewEnvironment()
|
||||||
if err := bbgo.BootstrapBacktestEnvironment(ctx, environ); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if environ.DatabaseService == nil {
|
if userConfig.Backtest.CsvSource == nil {
|
||||||
return errors.New("database service is not enabled, please check your environment variables DB_DRIVER and DB_DSN")
|
return fmt.Errorf("user config backtest section needs csvsource config")
|
||||||
}
|
}
|
||||||
|
backtestService := service.NewBacktestServiceCSV(
|
||||||
|
outputDirectory,
|
||||||
|
userConfig.Backtest.CsvSource.Market,
|
||||||
|
userConfig.Backtest.CsvSource.Granularity,
|
||||||
|
)
|
||||||
|
if modeCsv {
|
||||||
|
if err := bbgo.BootstrapEnvironmentLightweight(ctx, environ, userConfig); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
backtestService = service.NewBacktestService(environ.DatabaseService.DB)
|
||||||
|
if err := bbgo.BootstrapBacktestEnvironment(ctx, environ); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
backtestService := &service.BacktestService{DB: environ.DatabaseService.DB}
|
if environ.DatabaseService == nil {
|
||||||
|
return errors.New("database service is not enabled, please check your environment variables DB_DRIVER and DB_DSN")
|
||||||
|
}
|
||||||
|
}
|
||||||
environ.BacktestService = backtestService
|
environ.BacktestService = backtestService
|
||||||
bbgo.SetBackTesting(backtestService)
|
bbgo.SetBackTesting(backtestService)
|
||||||
|
|
||||||
|
@ -545,12 +565,11 @@ var BacktestCmd = &cobra.Command{
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
tradeState := sessionTradeStats[session.Name][symbol]
|
|
||||||
profitFactor := tradeState.ProfitFactor
|
profitFactor := tradeState.ProfitFactor
|
||||||
winningRatio := tradeState.WinningRatio
|
winningRatio := tradeState.WinningRatio
|
||||||
intervalProfits := tradeState.IntervalProfits[types.Interval1d]
|
intervalProfits := tradeState.IntervalProfits[types.Interval1d]
|
||||||
|
|
||||||
symbolReport, err := createSymbolReport(userConfig, session, symbol, trades.Copy(), tradeStats)
|
symbolReport, err := createSymbolReport(userConfig, session, symbol, trades.Copy(), intervalProfits, profitFactor, winningRatio)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -623,8 +642,6 @@ func createSymbolReport(
|
||||||
*backtest.SessionSymbolReport,
|
*backtest.SessionSymbolReport,
|
||||||
error,
|
error,
|
||||||
) {
|
) {
|
||||||
intervalProfit := tradeStats.IntervalProfits[types.Interval1d]
|
|
||||||
|
|
||||||
backtestExchange, ok := session.Exchange.(*backtest.Exchange)
|
backtestExchange, ok := session.Exchange.(*backtest.Exchange)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("unexpected error, exchange instance is not a backtest exchange")
|
return nil, fmt.Errorf("unexpected error, exchange instance is not a backtest exchange")
|
||||||
|
@ -634,11 +651,6 @@ func createSymbolReport(
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("market not found: %s, %s", symbol, session.Exchange.Name())
|
return nil, fmt.Errorf("market not found: %s, %s", symbol, session.Exchange.Name())
|
||||||
}
|
}
|
||||||
tStart, tEnd := trades[0].Time, trades[len(trades)-1].Time
|
|
||||||
|
|
||||||
periodStart := tStart.Time()
|
|
||||||
periodEnd := tEnd.Time()
|
|
||||||
period := periodEnd.Sub(periodStart)
|
|
||||||
|
|
||||||
startPrice, ok := session.StartPrice(symbol)
|
startPrice, ok := session.StartPrice(symbol)
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -655,81 +667,29 @@ func createSymbolReport(
|
||||||
Market: market,
|
Market: market,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sharpeRatio := fixedpoint.NewFromFloat(intervalProfit.GetSharpe())
|
||||||
|
sortinoRatio := fixedpoint.NewFromFloat(intervalProfit.GetSortino())
|
||||||
|
|
||||||
report := calculator.Calculate(symbol, trades, lastPrice)
|
report := calculator.Calculate(symbol, trades, lastPrice)
|
||||||
accountConfig := userConfig.Backtest.GetAccount(session.Exchange.Name().String())
|
accountConfig := userConfig.Backtest.GetAccount(session.Exchange.Name().String())
|
||||||
initBalances := accountConfig.Balances.BalanceMap()
|
initBalances := accountConfig.Balances.BalanceMap()
|
||||||
finalBalances := session.GetAccount().Balances()
|
finalBalances := session.GetAccount().Balances()
|
||||||
maxProfit := n(intervalProfit.Profits.Max())
|
|
||||||
maxLoss := n(intervalProfit.Profits.Min())
|
|
||||||
drawdown := types.Drawdown(intervalProfit.Profits)
|
|
||||||
maxDrawdown := drawdown.Max()
|
|
||||||
avgDrawdown := drawdown.Average()
|
|
||||||
roundTurnCount := n(float64(tradeStats.NumOfProfitTrade + tradeStats.NumOfLossTrade))
|
|
||||||
roundTurnLength := n(float64(intervalProfit.Profits.Length()))
|
|
||||||
winningCount := n(float64(tradeStats.NumOfProfitTrade))
|
|
||||||
loosingCount := n(float64(tradeStats.NumOfLossTrade))
|
|
||||||
avgProfit := tradeStats.GrossProfit.Div(n(types.NNZ(float64(tradeStats.NumOfProfitTrade), 1)))
|
|
||||||
avgLoss := tradeStats.GrossLoss.Div(n(types.NNZ(float64(tradeStats.NumOfLossTrade), 1)))
|
|
||||||
|
|
||||||
winningPct := winningCount.Div(roundTurnCount)
|
|
||||||
// losingPct := fixedpoint.One.Sub(winningPct)
|
|
||||||
|
|
||||||
sharpeRatio := n(intervalProfit.GetSharpe())
|
|
||||||
sortinoRatio := n(intervalProfit.GetSortino())
|
|
||||||
annVolHis := n(types.AnnualHistoricVolatility(intervalProfit.Profits))
|
|
||||||
totalTimeInMarketSec, avgHoldSec := intervalProfit.GetTimeInMarket()
|
|
||||||
statn, stdErr := types.StatN(intervalProfit.Profits)
|
|
||||||
symbolReport := backtest.SessionSymbolReport{
|
symbolReport := backtest.SessionSymbolReport{
|
||||||
Exchange: session.Exchange.Name(),
|
Exchange: session.Exchange.Name(),
|
||||||
Symbol: symbol,
|
Symbol: symbol,
|
||||||
Market: market,
|
Market: market,
|
||||||
LastPrice: lastPrice,
|
LastPrice: lastPrice,
|
||||||
StartPrice: startPrice,
|
StartPrice: startPrice,
|
||||||
InitialBalances: initBalances,
|
PnL: report,
|
||||||
FinalBalances: finalBalances,
|
InitialBalances: initBalances,
|
||||||
TradeCount: fixedpoint.NewFromInt(int64(len(trades))),
|
FinalBalances: finalBalances,
|
||||||
GrossLoss: tradeStats.GrossLoss,
|
// Manifests: manifests,
|
||||||
GrossProfit: tradeStats.GrossProfit,
|
Sharpe: sharpeRatio,
|
||||||
WinningCount: tradeStats.NumOfProfitTrade,
|
Sortino: sortinoRatio,
|
||||||
LosingCount: tradeStats.NumOfLossTrade,
|
ProfitFactor: profitFactor,
|
||||||
RoundTurnCount: roundTurnCount,
|
WinningRatio: winningRatio,
|
||||||
WinningRatio: tradeStats.WinningRatio,
|
|
||||||
PercentProfitable: winningPct,
|
|
||||||
ProfitFactor: tradeStats.ProfitFactor,
|
|
||||||
MaxDrawdown: n(maxDrawdown),
|
|
||||||
AverageDrawdown: n(avgDrawdown),
|
|
||||||
MaxProfit: maxProfit,
|
|
||||||
MaxLoss: maxLoss,
|
|
||||||
MaxLossStreak: tradeStats.MaximumConsecutiveLosses,
|
|
||||||
TotalTimeInMarketSec: totalTimeInMarketSec,
|
|
||||||
AvgHoldSec: avgHoldSec,
|
|
||||||
AvgProfit: avgProfit,
|
|
||||||
AvgLoss: avgLoss,
|
|
||||||
AvgNetProfit: tradeStats.TotalNetProfit.Div(roundTurnLength),
|
|
||||||
TotalNetProfit: tradeStats.TotalNetProfit,
|
|
||||||
AnnualHistoricVolatility: annVolHis,
|
|
||||||
PnL: report,
|
|
||||||
PRR: types.PRR(tradeStats.GrossProfit, tradeStats.GrossLoss, winningCount, loosingCount),
|
|
||||||
Kelly: types.KellyCriterion(tradeStats.ProfitFactor, winningPct),
|
|
||||||
OptimalF: types.OptimalF(intervalProfit.Profits),
|
|
||||||
StatN: statn,
|
|
||||||
StdErr: stdErr,
|
|
||||||
Sharpe: sharpeRatio,
|
|
||||||
Sortino: sortinoRatio,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cagr := types.NN(
|
|
||||||
types.CAGR(
|
|
||||||
symbolReport.InitialEquityValue().Float64(),
|
|
||||||
symbolReport.FinalEquityValue().Float64(),
|
|
||||||
int(period.Hours())/24,
|
|
||||||
), 0)
|
|
||||||
|
|
||||||
symbolReport.CAGR = n(cagr)
|
|
||||||
symbolReport.Calmar = n(types.CalmarRatio(cagr, maxDrawdown))
|
|
||||||
symbolReport.Sterling = n(types.SterlingRatio(cagr, avgDrawdown))
|
|
||||||
symbolReport.Burke = n(types.BurkeRatio(cagr, drawdown.AverageSquared()))
|
|
||||||
|
|
||||||
for _, s := range session.Subscriptions {
|
for _, s := range session.Subscriptions {
|
||||||
symbolReport.Subscriptions = append(symbolReport.Subscriptions, s)
|
symbolReport.Subscriptions = append(symbolReport.Subscriptions, s)
|
||||||
}
|
}
|
||||||
|
@ -748,12 +708,8 @@ func createSymbolReport(
|
||||||
return &symbolReport, nil
|
return &symbolReport, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func n(v float64) fixedpoint.Value {
|
|
||||||
return fixedpoint.NewFromFloat(v)
|
|
||||||
}
|
|
||||||
|
|
||||||
func verify(
|
func verify(
|
||||||
userConfig *bbgo.Config, backtestService *service.BacktestService,
|
userConfig *bbgo.Config, backtestService service.BackTestable,
|
||||||
sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time,
|
sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time,
|
||||||
) error {
|
) error {
|
||||||
for _, sourceExchange := range sourceExchanges {
|
for _, sourceExchange := range sourceExchanges {
|
||||||
|
@ -796,7 +752,7 @@ func getExchangeIntervals(ex types.Exchange) types.IntervalMap {
|
||||||
}
|
}
|
||||||
|
|
||||||
func sync(
|
func sync(
|
||||||
ctx context.Context, userConfig *bbgo.Config, backtestService *service.BacktestService,
|
ctx context.Context, userConfig *bbgo.Config, backtestService service.BackTestable,
|
||||||
sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time,
|
sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time,
|
||||||
) error {
|
) error {
|
||||||
for _, symbol := range userConfig.Backtest.Symbols {
|
for _, symbol := range userConfig.Backtest.Symbols {
|
||||||
|
@ -811,10 +767,8 @@ func sync(
|
||||||
var intervals = supportIntervals.Slice()
|
var intervals = supportIntervals.Slice()
|
||||||
intervals.Sort()
|
intervals.Sort()
|
||||||
|
|
||||||
for _, interval := range intervals {
|
if err := backtestService.Sync(ctx, sourceExchange, symbol, intervals, syncFrom, syncTo); err != nil {
|
||||||
if err := backtestService.Sync(ctx, sourceExchange, symbol, interval, syncFrom, syncTo); err != nil {
|
return err
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -135,7 +135,8 @@ var PnLCmd = &cobra.Command{
|
||||||
|
|
||||||
// we need the backtest klines for the daily prices
|
// we need the backtest klines for the daily prices
|
||||||
backtestService := &service.BacktestService{DB: environ.DatabaseService.DB}
|
backtestService := &service.BacktestService{DB: environ.DatabaseService.DB}
|
||||||
if err := backtestService.Sync(ctx, exchange, symbol, types.Interval1d, since, until); err != nil {
|
intervals := []types.Interval{types.Interval1d}
|
||||||
|
if err := backtestService.Sync(ctx, exchange, symbol, intervals, since, until); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
152
pkg/datasource/csvsource/csv_kline_decoder.go
Normal file
152
pkg/datasource/csvsource/csv_kline_decoder.go
Normal file
|
@ -0,0 +1,152 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/fixedpoint"
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MetaTraderTimeFormat is the time format expected by the MetaTrader decoder when cols [0] and [1] are used.
|
||||||
|
const MetaTraderTimeFormat = "02/01/2006 15:04"
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrNotEnoughColumns is returned when the CSV price record does not have enough columns.
|
||||||
|
ErrNotEnoughColumns = errors.New("not enough columns")
|
||||||
|
|
||||||
|
// ErrInvalidTimeFormat is returned when the CSV price record does not have a valid time unix milli format.
|
||||||
|
ErrInvalidIDFormat = errors.New("cannot parse trade id string")
|
||||||
|
|
||||||
|
// ErrInvalidBoolFormat is returned when the CSV isBuyerMaker record does not have a valid bool representation.
|
||||||
|
ErrInvalidBoolFormat = errors.New("cannot parse bool to string")
|
||||||
|
|
||||||
|
// ErrInvalidTimeFormat is returned when the CSV price record does not have a valid time unix milli format.
|
||||||
|
ErrInvalidTimeFormat = errors.New("cannot parse time string")
|
||||||
|
|
||||||
|
// ErrInvalidOrderSideFormat is returned when the CSV side record does not have a valid buy or sell string.
|
||||||
|
ErrInvalidOrderSideFormat = errors.New("cannot parse order side string")
|
||||||
|
|
||||||
|
// ErrInvalidPriceFormat is returned when the CSV price record does not prices in expected format.
|
||||||
|
ErrInvalidPriceFormat = errors.New("OHLC prices must be valid number format")
|
||||||
|
|
||||||
|
// ErrInvalidVolumeFormat is returned when the CSV price record does not have a valid volume format.
|
||||||
|
ErrInvalidVolumeFormat = errors.New("volume must be valid number format")
|
||||||
|
)
|
||||||
|
|
||||||
|
// CSVKLineDecoder is an extension point for CSVKLineReader to support custom file formats.
|
||||||
|
type CSVKLineDecoder func(record []string, interval time.Duration) (types.KLine, error)
|
||||||
|
|
||||||
|
// NewBinanceCSVKLineReader creates a new CSVKLineReader for Binance CSV files.
|
||||||
|
func NewBinanceCSVKLineReader(csv *csv.Reader) *CSVKLineReader {
|
||||||
|
return &CSVKLineReader{
|
||||||
|
csv: csv,
|
||||||
|
decoder: BinanceCSVKLineDecoder,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BinanceCSVKLineDecoder decodes a CSV record from Binance or Bybit into a KLine.
|
||||||
|
func BinanceCSVKLineDecoder(record []string, interval time.Duration) (types.KLine, error) {
|
||||||
|
var (
|
||||||
|
k, empty types.KLine
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
if len(record) < 5 {
|
||||||
|
return k, ErrNotEnoughColumns
|
||||||
|
}
|
||||||
|
ts, err := strconv.ParseFloat(record[0], 64) // check for e numbers "1.70027E+12"
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidTimeFormat
|
||||||
|
}
|
||||||
|
open, err := fixedpoint.NewFromString(record[1])
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
high, err := fixedpoint.NewFromString(record[2])
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
low, err := fixedpoint.NewFromString(record[3])
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
closing, err := fixedpoint.NewFromString(record[4])
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
volume, err := fixedpoint.NewFromString(record[5])
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidVolumeFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
k.StartTime = types.NewTimeFromUnix(int64(ts), 0)
|
||||||
|
k.EndTime = types.NewTimeFromUnix(k.StartTime.Time().Add(interval).Unix(), 0)
|
||||||
|
k.Open = open
|
||||||
|
k.High = high
|
||||||
|
k.Low = low
|
||||||
|
k.Close = closing
|
||||||
|
k.Volume = volume
|
||||||
|
|
||||||
|
return k, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMetaTraderCSVKLineReader creates a new CSVKLineReader for MetaTrader CSV files.
|
||||||
|
func NewMetaTraderCSVKLineReader(csv *csv.Reader) *CSVKLineReader {
|
||||||
|
csv.Comma = ';'
|
||||||
|
return &CSVKLineReader{
|
||||||
|
csv: csv,
|
||||||
|
decoder: MetaTraderCSVKLineDecoder,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MetaTraderCSVKLineDecoder decodes a CSV record from MetaTrader into a KLine.
|
||||||
|
func MetaTraderCSVKLineDecoder(record []string, interval time.Duration) (types.KLine, error) {
|
||||||
|
var (
|
||||||
|
k, empty types.KLine
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(record) < 6 {
|
||||||
|
return k, ErrNotEnoughColumns
|
||||||
|
}
|
||||||
|
|
||||||
|
tStr := fmt.Sprintf("%s %s", record[0], record[1])
|
||||||
|
t, err := time.Parse(MetaTraderTimeFormat, tStr)
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidTimeFormat
|
||||||
|
}
|
||||||
|
open, err := fixedpoint.NewFromString(record[2])
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
high, err := fixedpoint.NewFromString(record[3])
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
low, err := fixedpoint.NewFromString(record[4])
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
closing, err := fixedpoint.NewFromString(record[5])
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
volume, err := fixedpoint.NewFromString(record[6])
|
||||||
|
if err != nil {
|
||||||
|
return empty, ErrInvalidVolumeFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
k.StartTime = types.NewTimeFromUnix(t.Unix(), 0)
|
||||||
|
k.EndTime = types.NewTimeFromUnix(t.Add(interval).Unix(), 0)
|
||||||
|
k.Open = open
|
||||||
|
k.High = high
|
||||||
|
k.Low = low
|
||||||
|
k.Close = closing
|
||||||
|
k.Volume = volume
|
||||||
|
|
||||||
|
return k, nil
|
||||||
|
}
|
65
pkg/datasource/csvsource/csv_kline_reader.go
Normal file
65
pkg/datasource/csvsource/csv_kline_reader.go
Normal file
|
@ -0,0 +1,65 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"io"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
var _ KLineReader = (*CSVKLineReader)(nil)
|
||||||
|
|
||||||
|
// CSVKLineReader is a KLineReader that reads from a CSV file.
|
||||||
|
type CSVKLineReader struct {
|
||||||
|
csv *csv.Reader
|
||||||
|
decoder CSVKLineDecoder
|
||||||
|
}
|
||||||
|
|
||||||
|
// MakeCSVKLineReader is a factory method type that creates a new CSVKLineReader.
|
||||||
|
type MakeCSVKLineReader func(csv *csv.Reader) *CSVKLineReader
|
||||||
|
|
||||||
|
// NewCSVKLineReader creates a new CSVKLineReader with the default Binance decoder.
|
||||||
|
func NewCSVKLineReader(csv *csv.Reader) *CSVKLineReader {
|
||||||
|
return &CSVKLineReader{
|
||||||
|
csv: csv,
|
||||||
|
decoder: BinanceCSVKLineDecoder,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCSVKLineReaderWithDecoder creates a new CSVKLineReader with the given decoder.
|
||||||
|
func NewCSVKLineReaderWithDecoder(csv *csv.Reader, decoder CSVKLineDecoder) *CSVKLineReader {
|
||||||
|
return &CSVKLineReader{
|
||||||
|
csv: csv,
|
||||||
|
decoder: decoder,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read reads the next KLine from the underlying CSV data.
|
||||||
|
func (r *CSVKLineReader) Read(interval time.Duration) (types.KLine, error) {
|
||||||
|
var k types.KLine
|
||||||
|
|
||||||
|
rec, err := r.csv.Read()
|
||||||
|
if err != nil {
|
||||||
|
return k, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return r.decoder(rec, interval)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadAll reads all the KLines from the underlying CSV data.
|
||||||
|
func (r *CSVKLineReader) ReadAll(interval time.Duration) ([]types.KLine, error) {
|
||||||
|
var ks []types.KLine
|
||||||
|
for {
|
||||||
|
k, err := r.Read(interval)
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
ks = append(ks, k)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ks, nil
|
||||||
|
}
|
163
pkg/datasource/csvsource/csv_kline_reader_test.go
Normal file
163
pkg/datasource/csvsource/csv_kline_reader_test.go
Normal file
|
@ -0,0 +1,163 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/davecgh/go-spew/spew"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/fixedpoint"
|
||||||
|
. "github.com/c9s/bbgo/pkg/testing/testhelper"
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func assertKLineEq(t *testing.T, exp, act types.KLine, name string) {
|
||||||
|
assert.Equal(t, exp.StartTime, act.StartTime, name)
|
||||||
|
assert.Equal(t, 0, exp.Open.Compare(act.Open), name)
|
||||||
|
assert.Equal(t, 0, exp.High.Compare(act.High), name)
|
||||||
|
assert.Equal(t, 0, exp.Low.Compare(act.Low), name)
|
||||||
|
assert.Equal(t, 0, exp.Close.Compare(act.Close), name)
|
||||||
|
assert.Equal(t, 0, exp.Volume.Compare(act.Volume), name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCSVKLineReader_ReadWithBinanceDecoder(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
give string
|
||||||
|
want types.KLine
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Read DOHLCV",
|
||||||
|
give: "1609459200000,28923.63000000,29031.34000000,28690.17000000,28995.13000000,2311.81144500",
|
||||||
|
want: types.KLine{
|
||||||
|
StartTime: types.NewTimeFromUnix(1609459200, 0),
|
||||||
|
Open: Number(28923.63),
|
||||||
|
High: Number(29031.34),
|
||||||
|
Low: Number(28690.17),
|
||||||
|
Close: Number(28995.13),
|
||||||
|
// todo this should never happen >>
|
||||||
|
// mustNewFromString and NewFromFloat have different values after parse
|
||||||
|
Volume: fixedpoint.MustNewFromString("2311.81144500")},
|
||||||
|
err: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Read DOHLC",
|
||||||
|
give: "1609459200000,28923.63000000,29031.34000000,28690.17000000,28995.13000000",
|
||||||
|
want: types.KLine{
|
||||||
|
StartTime: types.NewTimeFromUnix(1609459200, 0),
|
||||||
|
Open: Number(28923.63),
|
||||||
|
High: Number(29031.34),
|
||||||
|
Low: Number(28690.17),
|
||||||
|
Close: Number(28995.13),
|
||||||
|
Volume: Number(0)},
|
||||||
|
err: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Not enough columns",
|
||||||
|
give: "1609459200000,28923.63000000,29031.34000000",
|
||||||
|
want: types.KLine{},
|
||||||
|
err: ErrNotEnoughColumns,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid time format",
|
||||||
|
give: "23/12/2021,28923.63000000,29031.34000000,28690.17000000,28995.13000000",
|
||||||
|
want: types.KLine{},
|
||||||
|
err: ErrInvalidTimeFormat,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid price format",
|
||||||
|
give: "1609459200000,sixty,29031.34000000,28690.17000000,28995.13000000",
|
||||||
|
want: types.KLine{},
|
||||||
|
err: ErrInvalidPriceFormat,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid volume format",
|
||||||
|
give: "1609459200000,28923.63000000,29031.34000000,28690.17000000,28995.13000000,vol",
|
||||||
|
want: types.KLine{},
|
||||||
|
err: ErrInvalidVolumeFormat,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
reader := NewBinanceCSVKLineReader(csv.NewReader(strings.NewReader(tt.give)))
|
||||||
|
kline, err := reader.Read(time.Hour)
|
||||||
|
assert.Equal(t, tt.err, err)
|
||||||
|
if err == nil {
|
||||||
|
spew.Dump(tt.want)
|
||||||
|
spew.Dump(kline)
|
||||||
|
assertKLineEq(t, tt.want, kline, tt.name)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCSVKLineReader_ReadAllWithDefaultDecoder(t *testing.T) {
|
||||||
|
records := []string{
|
||||||
|
"1609459200000,28923.63000000,29031.34000000,28690.17000000,28995.13000000,2311.81144500",
|
||||||
|
"1609459300000,28928.63000000,30031.34000000,22690.17000000,28495.13000000,3000.00",
|
||||||
|
}
|
||||||
|
reader := NewCSVKLineReader(csv.NewReader(strings.NewReader(strings.Join(records, "\n"))))
|
||||||
|
klines, err := reader.ReadAll(time.Hour)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Len(t, klines, 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCSVKLineReader_ReadWithMetaTraderDecoder(t *testing.T) {
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
give string
|
||||||
|
want types.KLine
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Read DOHLCV",
|
||||||
|
give: "11/12/2008;16:00;779.527679;780.964756;777.527679;779.964756;5",
|
||||||
|
want: types.KLine{
|
||||||
|
StartTime: types.NewTimeFromUnix(time.Date(2008, 12, 11, 16, 0, 0, 0, time.UTC).Unix(), 0),
|
||||||
|
Open: Number(779.527679),
|
||||||
|
High: Number(780.964756),
|
||||||
|
Low: Number(777.527679),
|
||||||
|
Close: Number(779.964756),
|
||||||
|
Volume: Number(5)},
|
||||||
|
err: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Not enough columns",
|
||||||
|
give: "1609459200000;28923.63000000;29031.34000000",
|
||||||
|
want: types.KLine{},
|
||||||
|
err: ErrNotEnoughColumns,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid time format",
|
||||||
|
give: "23/12/2021;t;28923.63000000;29031.34000000;28690.17000000;28995.13000000",
|
||||||
|
want: types.KLine{},
|
||||||
|
err: ErrInvalidTimeFormat,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid price format",
|
||||||
|
give: "11/12/2008;00:00;sixty;29031.34000000;28690.17000000;28995.13000000",
|
||||||
|
want: types.KLine{},
|
||||||
|
err: ErrInvalidPriceFormat,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid volume format",
|
||||||
|
give: "11/12/2008;00:00;779.527679;780.964756;777.527679;779.964756;vol",
|
||||||
|
want: types.KLine{},
|
||||||
|
err: ErrInvalidVolumeFormat,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
reader := NewMetaTraderCSVKLineReader(csv.NewReader(strings.NewReader(tt.give)))
|
||||||
|
kline, err := reader.Read(time.Hour)
|
||||||
|
assert.Equal(t, tt.err, err)
|
||||||
|
assertKLineEq(t, tt.want, kline, tt.name)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
167
pkg/datasource/csvsource/csv_tick_converter.go
Normal file
167
pkg/datasource/csvsource/csv_tick_converter.go
Normal file
|
@ -0,0 +1,167 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/fixedpoint"
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ICSVTickConverter interface {
|
||||||
|
CsvTickToKLine(tick *CsvTick) (closesKLine bool)
|
||||||
|
GetTicks() []*CsvTick
|
||||||
|
LatestKLine(interval types.Interval) (k *types.KLine)
|
||||||
|
GetKLineResults() map[types.Interval][]types.KLine
|
||||||
|
}
|
||||||
|
|
||||||
|
// CSVTickConverter takes a tick and internally converts it to a KLine slice
|
||||||
|
type CSVTickConverter struct {
|
||||||
|
ticks []*CsvTick
|
||||||
|
intervals []types.Interval
|
||||||
|
klines map[types.Interval][]types.KLine
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewCSVTickConverter(intervals []types.Interval) ICSVTickConverter {
|
||||||
|
return &CSVTickConverter{
|
||||||
|
ticks: []*CsvTick{},
|
||||||
|
intervals: intervals,
|
||||||
|
klines: make(map[types.Interval][]types.KLine),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CSVTickConverter) GetTicks() []*CsvTick {
|
||||||
|
return c.ticks
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CSVTickConverter) AddKLine(interval types.Interval, k types.KLine) {
|
||||||
|
c.klines[interval] = append(c.klines[interval], k)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetKLineResult returns the converted ticks as kLine of interval
|
||||||
|
func (c *CSVTickConverter) LatestKLine(interval types.Interval) (k *types.KLine) {
|
||||||
|
if _, ok := c.klines[interval]; !ok || len(c.klines[interval]) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &c.klines[interval][len(c.klines[interval])-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetKLineResults returns the converted ticks as kLine of all constructed intervals
|
||||||
|
func (c *CSVTickConverter) GetKLineResults() map[types.Interval][]types.KLine {
|
||||||
|
if len(c.klines) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return c.klines
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert ticks to KLine with interval
|
||||||
|
func (c *CSVTickConverter) CsvTickToKLine(tick *CsvTick) (closesKLine bool) {
|
||||||
|
for _, interval := range c.intervals {
|
||||||
|
var (
|
||||||
|
currentCandle = types.KLine{}
|
||||||
|
high = fixedpoint.Zero
|
||||||
|
low = fixedpoint.Zero
|
||||||
|
)
|
||||||
|
isOpen, t := c.detCandleStart(tick.Timestamp.Time(), interval)
|
||||||
|
if isOpen {
|
||||||
|
latestKline := c.LatestKLine(interval)
|
||||||
|
if latestKline != nil {
|
||||||
|
latestKline.Closed = true // k is pointer
|
||||||
|
closesKLine = true
|
||||||
|
c.addMissingKLines(interval, t)
|
||||||
|
}
|
||||||
|
c.AddKLine(interval, types.KLine{
|
||||||
|
Exchange: tick.Exchange,
|
||||||
|
Symbol: tick.Symbol,
|
||||||
|
Interval: interval,
|
||||||
|
StartTime: types.NewTimeFromUnix(t.Unix(), 0),
|
||||||
|
EndTime: types.NewTimeFromUnix(t.Add(interval.Duration()).Unix(), 0),
|
||||||
|
Open: tick.Price,
|
||||||
|
High: tick.Price,
|
||||||
|
Low: tick.Price,
|
||||||
|
Close: tick.Price,
|
||||||
|
Volume: tick.HomeNotional,
|
||||||
|
QuoteVolume: tick.ForeignNotional,
|
||||||
|
Closed: false,
|
||||||
|
})
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
currentCandle = c.klines[interval][len(c.klines[interval])-1]
|
||||||
|
|
||||||
|
if tick.Price.Compare(currentCandle.High) > 0 {
|
||||||
|
high = tick.Price
|
||||||
|
} else {
|
||||||
|
high = currentCandle.High
|
||||||
|
}
|
||||||
|
|
||||||
|
if tick.Price.Compare(currentCandle.Low) < 0 {
|
||||||
|
low = tick.Price
|
||||||
|
} else {
|
||||||
|
low = currentCandle.Low
|
||||||
|
}
|
||||||
|
|
||||||
|
c.klines[interval][len(c.klines[interval])-1] = types.KLine{
|
||||||
|
StartTime: currentCandle.StartTime,
|
||||||
|
EndTime: currentCandle.EndTime,
|
||||||
|
Exchange: tick.Exchange,
|
||||||
|
Symbol: tick.Symbol,
|
||||||
|
Interval: interval,
|
||||||
|
Open: currentCandle.Open,
|
||||||
|
High: high,
|
||||||
|
Low: low,
|
||||||
|
Close: tick.Price,
|
||||||
|
Volume: currentCandle.Volume.Add(tick.HomeNotional),
|
||||||
|
QuoteVolume: currentCandle.QuoteVolume.Add(tick.ForeignNotional),
|
||||||
|
Closed: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CSVTickConverter) detCandleStart(ts time.Time, interval types.Interval) (isOpen bool, t time.Time) {
|
||||||
|
if len(c.klines) == 0 {
|
||||||
|
return true, interval.Truncate(ts)
|
||||||
|
}
|
||||||
|
|
||||||
|
var end = c.LatestKLine(interval).EndTime.Time()
|
||||||
|
if ts.After(end) {
|
||||||
|
return true, end
|
||||||
|
}
|
||||||
|
|
||||||
|
return false, t
|
||||||
|
}
|
||||||
|
|
||||||
|
// appendMissingKLines appends an empty kline till startNext falls within a kline interval
|
||||||
|
func (c *CSVTickConverter) addMissingKLines(
|
||||||
|
interval types.Interval,
|
||||||
|
startNext time.Time,
|
||||||
|
) {
|
||||||
|
for {
|
||||||
|
last := c.LatestKLine(interval)
|
||||||
|
newEndTime := types.NewTimeFromUnix(
|
||||||
|
// one second is the smallest interval
|
||||||
|
last.EndTime.Time().Add(time.Duration(last.Interval.Seconds())*time.Second).Unix(),
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
if last.EndTime.Time().Before(startNext) {
|
||||||
|
c.AddKLine(interval, types.KLine{
|
||||||
|
StartTime: last.EndTime,
|
||||||
|
EndTime: newEndTime,
|
||||||
|
Exchange: last.Exchange,
|
||||||
|
Symbol: last.Symbol,
|
||||||
|
Interval: last.Interval,
|
||||||
|
Open: last.Close,
|
||||||
|
High: last.Close,
|
||||||
|
Low: last.Close,
|
||||||
|
Close: last.Close,
|
||||||
|
Volume: 0,
|
||||||
|
QuoteVolume: 0,
|
||||||
|
Closed: true,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
189
pkg/datasource/csvsource/csv_tick_decoder.go
Normal file
189
pkg/datasource/csvsource/csv_tick_decoder.go
Normal file
|
@ -0,0 +1,189 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/fixedpoint"
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CSVTickDecoder is an extension point for CSVTickReader to support custom file formats.
|
||||||
|
type CSVTickDecoder func(record []string, index int) (*CsvTick, error)
|
||||||
|
|
||||||
|
// NewBinanceCSVTickReader creates a new CSVTickReader for Binance CSV files.
|
||||||
|
func NewBinanceCSVTickReader(csv *csv.Reader) *CSVTickReader {
|
||||||
|
return &CSVTickReader{
|
||||||
|
csv: csv,
|
||||||
|
decoder: BinanceCSVTickDecoder,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BinanceCSVKLineDecoder decodes a CSV record from Binance into a CsvTick.
|
||||||
|
func BinanceCSVTickDecoder(row []string, _ int) (*CsvTick, error) {
|
||||||
|
if len(row) < 5 {
|
||||||
|
return nil, ErrNotEnoughColumns
|
||||||
|
}
|
||||||
|
// example csv row for some reason some properties are duplicated in their csv
|
||||||
|
// id, price, qty, base_qty, base_qty, time, is_buyer_maker, is_buyer_maker,
|
||||||
|
// 11782578,6.00000000,1.00000000,14974844,14974844,1698623884463,True
|
||||||
|
id, err := strconv.ParseUint(row[0], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidIDFormat
|
||||||
|
}
|
||||||
|
price, err := fixedpoint.NewFromString(row[1])
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
qty, err := fixedpoint.NewFromString(row[2])
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidVolumeFormat
|
||||||
|
}
|
||||||
|
baseQty, err := fixedpoint.NewFromString(row[3])
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidVolumeFormat
|
||||||
|
}
|
||||||
|
isBuyerMaker, err := strconv.ParseBool(row[6])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// isBuyerMaker=false trade will qualify as BUY.
|
||||||
|
side := types.SideTypeBuy
|
||||||
|
if isBuyerMaker {
|
||||||
|
side = types.SideTypeSell
|
||||||
|
}
|
||||||
|
n, err := strconv.ParseFloat(row[5], 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidTimeFormat
|
||||||
|
}
|
||||||
|
ts := time.Unix(int64(n), 0)
|
||||||
|
return &CsvTick{
|
||||||
|
TradeID: id,
|
||||||
|
Exchange: types.ExchangeBinance,
|
||||||
|
Side: side,
|
||||||
|
Size: qty,
|
||||||
|
Price: price,
|
||||||
|
IsBuyerMaker: isBuyerMaker,
|
||||||
|
HomeNotional: price.Mul(qty),
|
||||||
|
ForeignNotional: price.Mul(baseQty),
|
||||||
|
Timestamp: types.NewMillisecondTimestampFromInt(ts.UnixMilli()),
|
||||||
|
// Symbol: must be overwritten - info not in csv,
|
||||||
|
// TickDirection: would need to keep last tick in memory to compare tick direction,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBinanceCSVTickReader creates a new CSVTickReader for Bybit CSV files.
|
||||||
|
func NewBybitCSVTickReader(csv *csv.Reader) *CSVTickReader {
|
||||||
|
return &CSVTickReader{
|
||||||
|
csv: csv,
|
||||||
|
decoder: BybitCSVTickDecoder,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BybitCSVTickDecoder decodes a CSV record from Bybit into a CsvTick.
|
||||||
|
func BybitCSVTickDecoder(row []string, index int) (*CsvTick, error) {
|
||||||
|
// example csv row
|
||||||
|
// timestamp,symbol,side,size,price,tickDirection,trdMatchID,grossValue,homeNotional,foreignNotional
|
||||||
|
// 1649054912,FXSUSDT,Buy,0.01,38.32,PlusTick,9c30abaf-80ae-5ebf-9850-58fe7ed4bac8,3.832e+07,0.01,0.3832
|
||||||
|
if len(row) < 9 {
|
||||||
|
return nil, ErrNotEnoughColumns
|
||||||
|
}
|
||||||
|
if index == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
side, err := types.StrToSideType(row[2])
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidOrderSideFormat
|
||||||
|
}
|
||||||
|
size, err := fixedpoint.NewFromString(row[3])
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidVolumeFormat
|
||||||
|
}
|
||||||
|
price, err := fixedpoint.NewFromString(row[4])
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
hn, err := fixedpoint.NewFromString(row[8])
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidVolumeFormat
|
||||||
|
}
|
||||||
|
fn, err := fixedpoint.NewFromString(row[9])
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidVolumeFormat
|
||||||
|
}
|
||||||
|
n, err := strconv.ParseFloat(row[0], 64) // startTime eg 1696982287.4922
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidTimeFormat
|
||||||
|
}
|
||||||
|
ts := time.Unix(int64(n), 0)
|
||||||
|
return &CsvTick{
|
||||||
|
TradeID: uint64(index),
|
||||||
|
Symbol: row[1],
|
||||||
|
Exchange: types.ExchangeBybit,
|
||||||
|
Side: side,
|
||||||
|
Size: size,
|
||||||
|
Price: price,
|
||||||
|
HomeNotional: hn,
|
||||||
|
ForeignNotional: fn,
|
||||||
|
TickDirection: row[5], // todo does this seem promising to define for other exchanges too?
|
||||||
|
Timestamp: types.NewMillisecondTimestampFromInt(ts.UnixMilli()),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewOKExCSVTickReader creates a new CSVTickReader for OKEx CSV files.
|
||||||
|
func NewOKExCSVTickReader(csv *csv.Reader) *CSVTickReader {
|
||||||
|
return &CSVTickReader{
|
||||||
|
csv: csv,
|
||||||
|
decoder: OKExCSVTickDecoder,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// OKExCSVKLineDecoder decodes a CSV record from OKEx into a CsvTick.
|
||||||
|
func OKExCSVTickDecoder(row []string, index int) (*CsvTick, error) {
|
||||||
|
if len(row) < 5 {
|
||||||
|
return nil, ErrNotEnoughColumns
|
||||||
|
}
|
||||||
|
if index == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
// example csv row for OKeX
|
||||||
|
// trade_id, side, size, price, created_time
|
||||||
|
// 134642, sell, 6.2638 6.507 1.69975E+12
|
||||||
|
id, err := strconv.ParseInt(row[0], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidIDFormat
|
||||||
|
}
|
||||||
|
price, err := fixedpoint.NewFromString(row[3])
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidPriceFormat
|
||||||
|
}
|
||||||
|
qty, err := fixedpoint.NewFromString(row[2])
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidVolumeFormat
|
||||||
|
}
|
||||||
|
side := types.SideTypeBuy
|
||||||
|
isBuyerMaker := false
|
||||||
|
if row[1] == "sell" {
|
||||||
|
side = types.SideTypeSell
|
||||||
|
isBuyerMaker = true
|
||||||
|
}
|
||||||
|
n, err := strconv.ParseFloat(row[4], 64) // startTime
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrInvalidTimeFormat
|
||||||
|
}
|
||||||
|
ts := time.Unix(int64(n), 0)
|
||||||
|
return &CsvTick{
|
||||||
|
TradeID: uint64(id),
|
||||||
|
Exchange: types.ExchangeOKEx,
|
||||||
|
Side: side,
|
||||||
|
Size: qty,
|
||||||
|
Price: price,
|
||||||
|
IsBuyerMaker: isBuyerMaker,
|
||||||
|
HomeNotional: price.Mul(qty),
|
||||||
|
Timestamp: types.NewMillisecondTimestampFromInt(ts.UnixMilli()),
|
||||||
|
// ForeignNotional: // info not in csv
|
||||||
|
// Symbol: must be overwritten - info not in csv
|
||||||
|
// TickDirection: would need to keep last tick in memory to compare tick direction,
|
||||||
|
}, nil
|
||||||
|
}
|
66
pkg/datasource/csvsource/csv_tick_reader.go
Normal file
66
pkg/datasource/csvsource/csv_tick_reader.go
Normal file
|
@ -0,0 +1,66 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
var _ TickReader = (*CSVTickReader)(nil)
|
||||||
|
|
||||||
|
// CSVTickReader is a CSVTickReader that reads from a CSV file.
|
||||||
|
type CSVTickReader struct {
|
||||||
|
csv *csv.Reader
|
||||||
|
decoder CSVTickDecoder
|
||||||
|
ticks []*CsvTick
|
||||||
|
}
|
||||||
|
|
||||||
|
// MakeCSVTickReader is a factory method type that creates a new CSVTickReader.
|
||||||
|
type MakeCSVTickReader func(csv *csv.Reader) *CSVTickReader
|
||||||
|
|
||||||
|
// NewCSVKLineReader creates a new CSVKLineReader with the default Binance decoder.
|
||||||
|
func NewCSVTickReader(csv *csv.Reader) *CSVTickReader {
|
||||||
|
return &CSVTickReader{
|
||||||
|
csv: csv,
|
||||||
|
decoder: BinanceCSVTickDecoder,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCSVTickReaderWithDecoder creates a new CSVKLineReader with the given decoder.
|
||||||
|
func NewCSVTickReaderWithDecoder(csv *csv.Reader, decoder CSVTickDecoder) *CSVTickReader {
|
||||||
|
return &CSVTickReader{
|
||||||
|
csv: csv,
|
||||||
|
decoder: decoder,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadAll reads all the KLines from the underlying CSV data.
|
||||||
|
func (r *CSVTickReader) ReadAll() (ticks []*CsvTick, err error) {
|
||||||
|
var i int
|
||||||
|
for {
|
||||||
|
tick, err := r.Read(i)
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
i++ // used as jump logic inside decoder to skip csv headers in case
|
||||||
|
if tick == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
ticks = append(ticks, tick)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ticks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read reads the next KLine from the underlying CSV data.
|
||||||
|
func (r *CSVTickReader) Read(i int) (*CsvTick, error) {
|
||||||
|
rec, err := r.csv.Read()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return r.decoder(rec, i)
|
||||||
|
}
|
75
pkg/datasource/csvsource/csv_tick_reader_test.go
Normal file
75
pkg/datasource/csvsource/csv_tick_reader_test.go
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
|
. "github.com/c9s/bbgo/pkg/testing/testhelper"
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCSVTickReader_ReadWithBinanceDecoder(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
give string
|
||||||
|
want *CsvTick
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Read Tick",
|
||||||
|
give: "11782578,6.00000000,1.00000000,14974844,14974844,1698623884463,True,True",
|
||||||
|
want: &CsvTick{
|
||||||
|
Timestamp: types.NewMillisecondTimestampFromInt(1698623884463),
|
||||||
|
Size: Number(1),
|
||||||
|
Price: Number(6),
|
||||||
|
HomeNotional: Number(6),
|
||||||
|
},
|
||||||
|
err: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Not enough columns",
|
||||||
|
give: "1609459200000,28923.63000000,29031.34000000",
|
||||||
|
want: nil,
|
||||||
|
err: ErrNotEnoughColumns,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid time format",
|
||||||
|
give: "11782578,6.00000000,1.00000000,14974844,14974844,23/12/2021,True,True",
|
||||||
|
want: nil,
|
||||||
|
err: ErrInvalidTimeFormat,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid price format",
|
||||||
|
give: "11782578,sixty,1.00000000,14974844,14974844,1698623884463,True,True",
|
||||||
|
want: nil,
|
||||||
|
err: ErrInvalidPriceFormat,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid size format",
|
||||||
|
give: "11782578,1.00000000,one,14974844,14974844,1698623884463,True,True",
|
||||||
|
want: nil,
|
||||||
|
err: ErrInvalidVolumeFormat,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
reader := NewBinanceCSVTickReader(csv.NewReader(strings.NewReader(tt.give)))
|
||||||
|
tick, err := reader.Read(0)
|
||||||
|
if err == nil {
|
||||||
|
assertTickEqual(t, tt.want, tick)
|
||||||
|
}
|
||||||
|
assert.Equal(t, tt.err, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertTickEqual(t *testing.T, exp, act *CsvTick) {
|
||||||
|
assert.Equal(t, exp.Timestamp.Time(), act.Timestamp.Time())
|
||||||
|
assert.Equal(t, 0, exp.Price.Compare(act.Price))
|
||||||
|
assert.Equal(t, 0, exp.Size.Compare(act.Size))
|
||||||
|
assert.Equal(t, 0, exp.HomeNotional.Compare(act.HomeNotional))
|
||||||
|
}
|
59
pkg/datasource/csvsource/read_klines.go
Normal file
59
pkg/datasource/csvsource/read_klines.go
Normal file
|
@ -0,0 +1,59 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// KLineReader is an interface for reading candlesticks.
|
||||||
|
type KLineReader interface {
|
||||||
|
Read(interval time.Duration) (types.KLine, error)
|
||||||
|
ReadAll(interval time.Duration) ([]types.KLine, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadKLinesFromCSV reads all the .csv files in a given directory or a single file into a slice of KLines.
|
||||||
|
// Wraps a default CSVKLineReader with Binance decoder for convenience.
|
||||||
|
// For finer grained memory management use the base kline reader.
|
||||||
|
func ReadKLinesFromCSV(path string, interval time.Duration) ([]types.KLine, error) {
|
||||||
|
return ReadKLinesFromCSVWithDecoder(path, interval, MakeCSVKLineReader(NewBinanceCSVKLineReader))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadKLinesFromCSVWithDecoder permits using a custom CSVKLineReader.
|
||||||
|
func ReadKLinesFromCSVWithDecoder(path string, interval time.Duration, maker MakeCSVKLineReader) ([]types.KLine, error) {
|
||||||
|
var klines []types.KLine
|
||||||
|
|
||||||
|
err := filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if filepath.Ext(path) != ".csv" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
//nolint:errcheck // Read ops only so safe to ignore err return
|
||||||
|
defer file.Close()
|
||||||
|
reader := maker(csv.NewReader(file))
|
||||||
|
newKlines, err := reader.ReadAll(interval)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
klines = append(klines, newKlines...)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return klines, nil
|
||||||
|
}
|
21
pkg/datasource/csvsource/read_klines_test.go
Normal file
21
pkg/datasource/csvsource/read_klines_test.go
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestReadKLinesFromCSV(t *testing.T) {
|
||||||
|
klines, err := ReadKLinesFromCSV("./testdata/binance/BTCUSDT-1h-2023-11-18.csv", time.Hour)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Len(t, klines, 24)
|
||||||
|
assert.Equal(t, int64(1700265600), klines[0].StartTime.Unix(), "StartTime")
|
||||||
|
assert.Equal(t, int64(1700269200), klines[0].EndTime.Unix(), "EndTime")
|
||||||
|
assert.Equal(t, 36613.91, klines[0].Open.Float64(), "Open")
|
||||||
|
assert.Equal(t, 36613.92, klines[0].High.Float64(), "High")
|
||||||
|
assert.Equal(t, 36388.12, klines[0].Low.Float64(), "Low")
|
||||||
|
assert.Equal(t, 36400.01, klines[0].Close.Float64(), "Close")
|
||||||
|
assert.Equal(t, 1005.75727, klines[0].Volume.Float64(), "Volume")
|
||||||
|
}
|
89
pkg/datasource/csvsource/read_ticks.go
Normal file
89
pkg/datasource/csvsource/read_ticks.go
Normal file
|
@ -0,0 +1,89 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TickReader is an interface for reading candlesticks.
|
||||||
|
type TickReader interface {
|
||||||
|
Read(i int) (*CsvTick, error)
|
||||||
|
ReadAll() (ticks []*CsvTick, err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadTicksFromCSV reads all the .csv files in a given directory or a single file into a slice of Ticks.
|
||||||
|
// Wraps a default CSVTickReader with Binance decoder for convenience.
|
||||||
|
// For finer grained memory management use the base kline reader.
|
||||||
|
func ReadTicksFromCSV(
|
||||||
|
path, symbol string,
|
||||||
|
intervals []types.Interval,
|
||||||
|
) (
|
||||||
|
klineMap map[types.Interval][]types.KLine,
|
||||||
|
err error,
|
||||||
|
) {
|
||||||
|
return ReadTicksFromCSVWithDecoder(
|
||||||
|
path,
|
||||||
|
symbol,
|
||||||
|
intervals,
|
||||||
|
MakeCSVTickReader(NewBinanceCSVTickReader),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadTicksFromCSVWithDecoder permits using a custom CSVTickReader.
|
||||||
|
func ReadTicksFromCSVWithDecoder(
|
||||||
|
path, symbol string,
|
||||||
|
intervals []types.Interval,
|
||||||
|
maker MakeCSVTickReader,
|
||||||
|
) (
|
||||||
|
klineMap map[types.Interval][]types.KLine,
|
||||||
|
err error,
|
||||||
|
) {
|
||||||
|
converter := NewCSVTickConverter(intervals)
|
||||||
|
ticks := []*CsvTick{}
|
||||||
|
// read all ticks into memory
|
||||||
|
err = filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if filepath.Ext(path) != ".csv" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
//nolint:errcheck // Read ops only so safe to ignore err return
|
||||||
|
defer file.Close()
|
||||||
|
reader := maker(csv.NewReader(file))
|
||||||
|
newTicks, err := reader.ReadAll()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ticks = append(ticks, newTicks...)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// sort ticks by timestamp (okex sorts csv by price ascending ;(
|
||||||
|
sort.Slice(ticks, func(i, j int) bool {
|
||||||
|
return ticks[i].Timestamp.Time().Before(ticks[j].Timestamp.Time())
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, tick := range ticks {
|
||||||
|
tick.Symbol = symbol
|
||||||
|
converter.CsvTickToKLine(tick)
|
||||||
|
}
|
||||||
|
|
||||||
|
return converter.GetKLineResults(), nil
|
||||||
|
}
|
67
pkg/datasource/csvsource/read_ticks_test.go
Normal file
67
pkg/datasource/csvsource/read_ticks_test.go
Normal file
|
@ -0,0 +1,67 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestReadTicksFromBinanceCSV(t *testing.T) {
|
||||||
|
path := "./testdata/binance/FXSUSDT-ticks-2023-10-29.csv"
|
||||||
|
symbol := "FXSUSDT"
|
||||||
|
intervals := []types.Interval{types.Interval1h}
|
||||||
|
klineMap, err := ReadTicksFromCSVWithDecoder(
|
||||||
|
path, symbol, intervals, MakeCSVTickReader(NewBinanceCSVTickReader),
|
||||||
|
)
|
||||||
|
klines := klineMap[types.Interval1h]
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Len(t, klines, 1)
|
||||||
|
assert.Equal(t, int64(1698620400), klines[0].StartTime.Unix(), "StartTime")
|
||||||
|
assert.Equal(t, int64(1698624000), klines[0].EndTime.Unix(), "EndTime")
|
||||||
|
assert.Equal(t, 6.0, klines[0].Open.Float64(), "Open")
|
||||||
|
assert.Equal(t, 6.0, klines[0].High.Float64(), "High")
|
||||||
|
assert.Equal(t, 6.0, klines[0].Low.Float64(), "Low")
|
||||||
|
assert.Equal(t, 6.0, klines[0].Close.Float64(), "Close")
|
||||||
|
assert.Equal(t, 111.0, klines[0].Volume.Float64(), "Volume")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadTicksFromBybitCSV(t *testing.T) {
|
||||||
|
path := "./testdata/bybit/FXSUSDT2023-10-10.csv"
|
||||||
|
symbol := "FXSUSDT"
|
||||||
|
intervals := []types.Interval{types.Interval1h}
|
||||||
|
klineMap, err := ReadTicksFromCSVWithDecoder(
|
||||||
|
path, symbol, intervals, MakeCSVTickReader(NewBybitCSVTickReader),
|
||||||
|
)
|
||||||
|
klines := klineMap[types.Interval1h]
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Len(t, klines, 1)
|
||||||
|
assert.Equal(t, int64(1696978800), klines[0].StartTime.Unix(), "StartTime")
|
||||||
|
assert.Equal(t, int64(1696982400), klines[0].EndTime.Unix(), "EndTime")
|
||||||
|
assert.Equal(t, 5.239, klines[0].Open.Float64(), "Open")
|
||||||
|
assert.Equal(t, 5.2495, klines[0].High.Float64(), "High")
|
||||||
|
assert.Equal(t, 5.239, klines[0].Low.Float64(), "Low")
|
||||||
|
assert.Equal(t, 5.2495, klines[0].Close.Float64(), "Close")
|
||||||
|
assert.Equal(t, 147.05, klines[0].Volume.Float64(), "Volume")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadTicksFromOkexCSV(t *testing.T) {
|
||||||
|
path := "./testdata/okex/BTC-USDT-aggtrades-2023-11-18.csv"
|
||||||
|
symbol := "BTCUSDT"
|
||||||
|
intervals := []types.Interval{types.Interval1h}
|
||||||
|
klineMap, err := ReadTicksFromCSVWithDecoder(
|
||||||
|
path, symbol, intervals, MakeCSVTickReader(NewOKExCSVTickReader),
|
||||||
|
)
|
||||||
|
klines := klineMap[types.Interval1h]
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Len(t, klines, 1)
|
||||||
|
assert.Equal(t, int64(1700236800), klines[0].StartTime.Unix(), "StartTime")
|
||||||
|
assert.Equal(t, int64(1700240400), klines[0].EndTime.Unix(), "EndTime")
|
||||||
|
assert.Equal(t, 35910.6, klines[0].Open.Float64(), "Open")
|
||||||
|
assert.Equal(t, 35914.4, klines[0].High.Float64(), "High")
|
||||||
|
assert.Equal(t, 35910.6, klines[0].Low.Float64(), "Low")
|
||||||
|
assert.Equal(t, 35914.4, klines[0].Close.Float64(), "Close")
|
||||||
|
assert.Equal(t, 51525.38700081, klines[0].Volume.Float64(), "Volume")
|
||||||
|
|
||||||
|
}
|
24
pkg/datasource/csvsource/testdata/binance/BTCUSDT-1h-2023-11-18.csv
vendored
Normal file
24
pkg/datasource/csvsource/testdata/binance/BTCUSDT-1h-2023-11-18.csv
vendored
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
1.70027E+12,36613.91,36613.92,36388.12,36400.01,1005.75727,1.70027E+12,36712312.49831390,35985,440.81212,16088534.76584510,0
|
||||||
|
1.70027E+12,36400.01,36456.53,36377.88,36405.46,507.51514,1.70027E+12,18486014.80771630,25492,236.98883,8631509.902,0
|
||||||
|
1.70027E+12,36405.47,36447.75,36390.44,36408.09,341.53256,1.70028E+12,12438100.14362490,19727,163.41072,5951076.256,0
|
||||||
|
1.70028E+12,36408.1,36424.01,36360.22,36371.81,444.73045,1.70028E+12,16180477.67130620,25089,207.88416,7562956.27,0
|
||||||
|
1.70028E+12,36371.8,36426.51,36369.45,36369.45,378.50007,1.70028E+12,13775839.75999540,20728,197.99667,7205858.497,0
|
||||||
|
1.70028E+12,36369.45,36378.65,36303.98,36334,629.09574,1.70029E+12,22862757.37912180,33883,269.3097,9787132.913,0
|
||||||
|
1.70029E+12,36334.01,36361.1,36250.01,36252,615.52755,1.70029E+12,22350527.58640450,30392,238.90543,8675479.987,0
|
||||||
|
1.70029E+12,36251.99,36428,36178.58,36417.16,1191.24433,1.70029E+12,43265058.27238300,41466,628.67499,22834722.23764540,0
|
||||||
|
1.70029E+12,36417.15,36479.22,36375.28,36448.01,600.66262,1.7003E+12,21883116.44525150,29227,301.84047,10996141.34025750,0
|
||||||
|
1.7003E+12,36448,36453.09,36392,36397.45,398.07607,1.7003E+12,14499345.43090060,22193,159.60456,5813290.376,0
|
||||||
|
1.7003E+12,36397.44,36486.48,36397.44,36472.46,601.46574,1.70031E+12,21917527.53081410,24881,354.42545,12916946.80705190,0
|
||||||
|
1.70031E+12,36472.46,36538.61,36400,36402.8,549.76216,1.70031E+12,20053594.27145890,29706,248.4342,9062453.31,0
|
||||||
|
1.70031E+12,36402.79,36484.31,36393.44,36449.3,513.24545,1.70031E+12,18705069.10380380,26631,244.2024,8898609.715,0
|
||||||
|
1.70031E+12,36449.3,36483.13,36347.69,36430.21,887.7206,1.70032E+12,32327899.78688460,41973,391.19851,14246544.95513180,0
|
||||||
|
1.70032E+12,36430.21,36568.76,36421.1,36507.03,803.12819,1.70032E+12,29307346.75876810,36941,447.83113,16341815.04367800,0
|
||||||
|
1.70032E+12,36507.04,36682.2,36505.14,36664.16,1440.91018,1.70032E+12,52738306.52534310,50174,755.06676,27635771.88129150,0
|
||||||
|
1.70032E+12,36664.17,36845.49,36639.96,36674,1669.58835,1.70033E+12,61326939.61543430,61313,823.2455,30239331.84409360,0
|
||||||
|
1.70033E+12,36674,36701.76,36600.1,36620,933.50168,1.70033E+12,34203500.13911480,39514,402.07999,14731130.96536590,0
|
||||||
|
1.70033E+12,36620.01,36745.5,36611.47,36707.19,583.0753,1.70033E+12,21373512.11709470,29144,289.6688,10617175.15516390,0
|
||||||
|
1.70033E+12,36707.18,36768,36653.51,36679.1,598.67548,1.70034E+12,21974467.38315690,31067,277.3739,10180364.08398050,0
|
||||||
|
1.70034E+12,36679.09,36707.16,36536.08,36598,779.88183,1.70034E+12,28546655.20047820,43054,314.94592,11526766.07883920,0
|
||||||
|
1.70034E+12,36597.99,36611.78,36524.31,36542.01,581.49791,1.70034E+12,21265094.72761260,36319,230.4279,8425965.352,0
|
||||||
|
1.70034E+12,36542.01,36556.56,36443.1,36506,543.12589,1.70035E+12,19821336.79152930,27472,245.46068,8956420.024,0
|
||||||
|
1.70035E+12,36506,36571.33,36498.34,36568.1,504.0213,1.70035E+12,18419269.57915190,22127,231.75877,8469107.669,0
|
|
5
pkg/datasource/csvsource/testdata/binance/FXSUSDT-ticks-2023-10-29.csv
vendored
Normal file
5
pkg/datasource/csvsource/testdata/binance/FXSUSDT-ticks-2023-10-29.csv
vendored
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
11782578,6.00000000,1.00000000,14974844,14974844,1698623884463,True,True
|
||||||
|
11782579,6.00000000,1.00000000,14974845,14974845,1698623884666,True,True
|
||||||
|
11782580,6.00000000,1.00000000,14974846,14974846,1698623893793,True,True
|
||||||
|
11782581,6.00000000,5.00000000,14974847,14974847,1698623920955,True,True
|
||||||
|
11782582,6.00000000,10.50000000,14974848,14974848,1698623939783,False,True
|
|
16
pkg/datasource/csvsource/testdata/bybit/FXSUSDT-ticks-2023-10-10.csv
vendored
Normal file
16
pkg/datasource/csvsource/testdata/bybit/FXSUSDT-ticks-2023-10-10.csv
vendored
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
timestamp,symbol,side,size,price,tickDirection,trdMatchID,grossValue,homeNotional,foreignNotional
|
||||||
|
1696982287.4922,FXSUSDT,Sell,0.86,5.2390,ZeroMinusTick,f7496ecb-b174-51b9-ba56-150186ba6c27,4.50554e+08,0.86,4.50554
|
||||||
|
1696982322.0561,FXSUSDT,Buy,0.13,5.2395,PlusTick,2089f1f4-d890-5762-a652-49a743fab436,6.81135e+07,0.13,0.6811349999999999
|
||||||
|
1696982333.0308,FXSUSDT,Buy,48.9,5.2420,PlusTick,8e7d405a-0003-5aa1-972d-46b08fe520c0,2.563338e+10,48.9,256.3338
|
||||||
|
1696982333.0377,FXSUSDT,Buy,0.77,5.2425,PlusTick,9f250e94-da5b-5a94-9126-084e46c9c692,4.0367249999999994e+08,0.77,4.036725
|
||||||
|
1696982359.7441,FXSUSDT,Buy,0.12,5.2450,PlusTick,08a0c666-da06-53f6-8eec-9d3462582b4f,6.293999999999999e+07,0.12,0.6294
|
||||||
|
1696982359.7441,FXSUSDT,Buy,0.19,5.2450,ZeroMinusTick,8a61753b-2a8e-5881-8e9b-9ad66806ee23,9.9655e+07,0.19,0.99655
|
||||||
|
1696982359.7443,FXSUSDT,Buy,12.12,5.2450,ZeroMinusTick,34b2f272-2f68-5d0a-a4ad-6c02f5342ca1,6.356939999999999e+09,12.12,63.569399999999995
|
||||||
|
1696982359.7443,FXSUSDT,Buy,2.19,5.2450,ZeroMinusTick,0cae9717-0fe1-51dd-bb10-1a61d5e83d98,1.148655e+09,2.19,11.48655
|
||||||
|
1696982359.7449,FXSUSDT,Buy,35.66,5.2450,ZeroMinusTick,0a5f0734-af3a-5439-9f17-d98ce7ea4f24,1.870367e+10,35.66,187.0367
|
||||||
|
1696982359.7512,FXSUSDT,Buy,10.97,5.2450,ZeroMinusTick,d8529e38-d3f5-5a7a-97f7-55cf3335de77,5.753765000000001e+09,10.97,57.537650000000006
|
||||||
|
1696982359.7512,FXSUSDT,Buy,22.97,5.2450,ZeroMinusTick,44361b86-78e1-533b-a3d0-7dd12d538992,1.2047765e+10,22.97,120.47765
|
||||||
|
1696982369.5962,FXSUSDT,Buy,0.05,5.2470,PlusTick,6800a047-b6e5-520a-9817-eb4d463a3cce,2.6235000000000004e+07,0.05,0.26235
|
||||||
|
1696982389.6288,FXSUSDT,Buy,0.02,5.2495,PlusTick,a4bc238f-3e6a-58a3-a012-1342563c2ced,1.0499000000000002e+07,0.02,0.10499000000000001
|
||||||
|
1696982389.6288,FXSUSDT,Buy,6.06,5.2495,ZeroMinusTick,eb27200e-c34e-537a-a0a0-4636dab66f07,3.181197e+09,6.06,31.81197
|
||||||
|
1696982389.6297,FXSUSDT,Buy,6.04,5.2495,ZeroMinusTick,c6badf81-05c5-5b35-b932-3c71941340fb,3.170698e+09,6.04,31.70698
|
|
16
pkg/datasource/csvsource/testdata/okex/BTC-USDT-aggtrades-2023-11-18.csv
vendored
Normal file
16
pkg/datasource/csvsource/testdata/okex/BTC-USDT-aggtrades-2023-11-18.csv
vendored
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
trade_id/<2F><><EFBFBD>id,side/<2F><><EFBFBD><EFBFBD><D7B7><EFBFBD>,size/<2F><><EFBFBD><EFBFBD>,price/<2F>۸<EFBFBD>,created_time/<2F>ɽ<EFBFBD>ʱ<EFBFBD><CAB1>
|
||||||
|
450372093,buy,0.00418025,35910.6,1700239042832
|
||||||
|
450372094,buy,0.0104,35911.9,1700239043163
|
||||||
|
450372860,buy,0.17316796,35911.9,1700239133047
|
||||||
|
450372095,buy,0.2227,35912.3,1700239043283
|
||||||
|
450372874,buy,0.63393,35913.4,1700239135563
|
||||||
|
450372876,buy,0.01751154,35913.6,1700239135563
|
||||||
|
450372096,buy,0.0478082,35913.7,1700239043339
|
||||||
|
450372877,buy,0.00030629,35913.7,1700239135563
|
||||||
|
450372878,buy,0.00030629,35913.8,1700239135563
|
||||||
|
450372880,buy,0.32111425,35913.9,1700239135563
|
||||||
|
450372881,buy,0.00027844,35914.0,1700239135563
|
||||||
|
450372882,buy,0.00058473,35914.2,1700239135563
|
||||||
|
450372032,buy,0.00132007,35914.3,1700239040621
|
||||||
|
450372883,buy,0.00058473,35914.3,1700239135563
|
||||||
|
450372884,buy,0.00052904,35914.4,1700239135563
|
|
280
pkg/datasource/csvsource/tick_downloader.go
Normal file
280
pkg/datasource/csvsource/tick_downloader.go
Normal file
|
@ -0,0 +1,280 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/exchange/kucoin"
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MarketType string
|
||||||
|
type DataType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
SPOT MarketType = "spot"
|
||||||
|
FUTURES MarketType = "futures"
|
||||||
|
TRADES DataType = "trades"
|
||||||
|
AGGTRADES DataType = "aggTrades"
|
||||||
|
// todo could be extended to:
|
||||||
|
|
||||||
|
// LEVEL2 = 2
|
||||||
|
// https://data.binance.vision/data/futures/um/daily/bookTicker/ADAUSDT/ADAUSDT-bookTicker-2023-11-19.zip
|
||||||
|
// update_id best_bid_price best_bid_qty best_ask_price best_ask_qty transaction_time event_time
|
||||||
|
// 3.52214E+12 0.3772 1632 0.3773 67521 1.70035E+12 1.70035E+12
|
||||||
|
|
||||||
|
// METRICS = 3
|
||||||
|
// https://data.binance.vision/data/futures/um/daily/metrics/ADAUSDT/ADAUSDT-metrics-2023-11-19.zip
|
||||||
|
// create_time symbol sum_open_interest sum_open_interest_value count_toptrader_long_short_ratio sum_toptrader_long_short_ratio count_long_short_ratio sum_taker_long_short_vol_ratio
|
||||||
|
// 19/11/2023 00:00 ADAUSDT 141979878.00000000 53563193.89339590 2.33412322 1.21401178 2.46604727 0.55265805
|
||||||
|
|
||||||
|
// KLINES DataType = 4
|
||||||
|
// https://public.bybit.com/kline_for_metatrader4/BNBUSDT/2021/BNBUSDT_15_2021-07-01_2021-07-31.csv.gz
|
||||||
|
// only few symbols but supported interval options 1m/ 5m/ 15m/ 30m/ 60m/ and only monthly
|
||||||
|
|
||||||
|
// https://data.binance.vision/data/futures/um/daily/klines/1INCHBTC/30m/1INCHBTC-30m-2023-11-18.zip
|
||||||
|
// supported interval options 1s/ 1m/ 3m/ 5m/ 15m/ 30m/ 1h/ 2h/ 4h/ 6h/ 8h/ 12h/ 1d/ daily or monthly futures
|
||||||
|
|
||||||
|
// this might be useful for backtesting against mark or index price
|
||||||
|
// especially index price can be used across exchanges
|
||||||
|
// https://data.binance.vision/data/futures/um/daily/indexPriceKlines/ADAUSDT/1h/ADAUSDT-1h-2023-11-19.zip
|
||||||
|
// https://data.binance.vision/data/futures/um/daily/markPriceKlines/ADAUSDT/1h/ADAUSDT-1h-2023-11-19.zip
|
||||||
|
|
||||||
|
// OKex or Bybit do not support direct kLine, metrics or level2 csv download
|
||||||
|
|
||||||
|
)
|
||||||
|
|
||||||
|
func Download(
|
||||||
|
path, symbol string,
|
||||||
|
exchange types.ExchangeName,
|
||||||
|
market MarketType,
|
||||||
|
granularity DataType,
|
||||||
|
since, until time.Time,
|
||||||
|
) (err error) {
|
||||||
|
for {
|
||||||
|
var (
|
||||||
|
fileName = fmt.Sprintf("%s-%s.csv", symbol, since.Format("2006-01-02"))
|
||||||
|
)
|
||||||
|
|
||||||
|
if fileExists(filepath.Join(path, fileName)) {
|
||||||
|
since = since.AddDate(0, 0, 1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var url, err = buildURL(exchange, symbol, market, granularity, fileName, since)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("fetching ", url)
|
||||||
|
|
||||||
|
csvContent, err := readCSVFromUrl(exchange, url)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
err = write(csvContent, fmt.Sprintf("%s/%s", path, granularity), fileName)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
since = since.AddDate(0, 0, 1)
|
||||||
|
if since.After(until) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildURL(
|
||||||
|
exchange types.ExchangeName,
|
||||||
|
symbol string,
|
||||||
|
market MarketType,
|
||||||
|
granularity DataType,
|
||||||
|
fileName string,
|
||||||
|
start time.Time,
|
||||||
|
) (url string, err error) {
|
||||||
|
switch exchange {
|
||||||
|
case types.ExchangeBybit:
|
||||||
|
// bybit doesn't seem to differentiate between spot and futures market or trade type in their csv dumps ;(
|
||||||
|
url = fmt.Sprintf("https://public.bybit.com/trading/%s/%s%s.csv.gz",
|
||||||
|
symbol,
|
||||||
|
symbol,
|
||||||
|
start.Format("2006-01-02"),
|
||||||
|
)
|
||||||
|
case types.ExchangeBinance:
|
||||||
|
marketType := "spot"
|
||||||
|
if market == FUTURES {
|
||||||
|
marketType = "futures/um"
|
||||||
|
}
|
||||||
|
dataType := "aggTrades"
|
||||||
|
if granularity == TRADES {
|
||||||
|
dataType = "trades"
|
||||||
|
}
|
||||||
|
url = fmt.Sprintf("https://data.binance.vision/data/%s/daily/%s/%s/%s-%s-%s.zip",
|
||||||
|
marketType,
|
||||||
|
dataType,
|
||||||
|
symbol,
|
||||||
|
symbol,
|
||||||
|
dataType,
|
||||||
|
start.Format("2006-01-02"))
|
||||||
|
|
||||||
|
case types.ExchangeOKEx:
|
||||||
|
// todo temporary find a better solution ?!
|
||||||
|
coins := strings.Split(kucoin.ToLocalSymbol(symbol), "-")
|
||||||
|
if len(coins) == 0 {
|
||||||
|
err = fmt.Errorf("%s not supported yet for OKEx.. care to fix it? PR's welcome ;)", symbol)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
baseCoin := coins[0]
|
||||||
|
quoteCoin := coins[1]
|
||||||
|
marketType := "" // for spot market
|
||||||
|
if market == FUTURES {
|
||||||
|
marketType = "-SWAP"
|
||||||
|
}
|
||||||
|
dataType := "aggtrades"
|
||||||
|
if granularity == TRADES {
|
||||||
|
dataType = "trades"
|
||||||
|
}
|
||||||
|
url = fmt.Sprintf("https://static.okx.com/cdn/okex/traderecords/%s/daily/%s/%s-%s%s-%s-%s.zip",
|
||||||
|
dataType,
|
||||||
|
start.Format("20060102"),
|
||||||
|
baseCoin,
|
||||||
|
quoteCoin,
|
||||||
|
marketType,
|
||||||
|
dataType,
|
||||||
|
start.Format("2006-01-02"))
|
||||||
|
default:
|
||||||
|
err = fmt.Errorf("%s not supported yet as csv data source.. care to fix it? PR's welcome ;)", exchange.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
return url, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func readCSVFromUrl(exchange types.ExchangeName, url string) (csvContent []byte, err error) {
|
||||||
|
resp, err := http.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("http get error, url %s: %w", url, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("unable to read response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch exchange {
|
||||||
|
case types.ExchangeBybit:
|
||||||
|
csvContent, err = gunzip(body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("gunzip data %s: %w", exchange, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
case types.ExchangeBinance:
|
||||||
|
csvContent, err = unzip(body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("unzip data %s: %w", exchange, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
case types.ExchangeOKEx:
|
||||||
|
csvContent, err = unzip(body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("unzip data %s: %w", exchange, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return csvContent, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func write(content []byte, path, fileName string) error {
|
||||||
|
|
||||||
|
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
|
||||||
|
err := os.MkdirAll(path, os.ModePerm)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("mkdir %s: %w", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dest := filepath.Join(path, fileName)
|
||||||
|
|
||||||
|
err := os.WriteFile(dest, content, 0666)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("write %s: %w", dest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func unzip(data []byte) (resData []byte, err error) {
|
||||||
|
zipReader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if zipReader == nil || len(zipReader.File) == 0 {
|
||||||
|
return nil, errors.New("no data to unzip")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read all the files from zip archive
|
||||||
|
for _, zipFile := range zipReader.File {
|
||||||
|
resData, err = readZipFile(zipFile)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func readZipFile(zf *zip.File) ([]byte, error) {
|
||||||
|
f, err := zf.Open()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
return io.ReadAll(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
func gunzip(data []byte) (resData []byte, err error) {
|
||||||
|
b := bytes.NewBuffer(data)
|
||||||
|
|
||||||
|
var r io.Reader
|
||||||
|
r, err = gzip.NewReader(b)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var resB bytes.Buffer
|
||||||
|
_, err = resB.ReadFrom(r)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
resData = resB.Bytes()
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileExists(fileName string) bool {
|
||||||
|
info, err := os.Stat(fileName)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return !info.IsDir()
|
||||||
|
}
|
103
pkg/datasource/csvsource/tick_downloader_test.go
Normal file
103
pkg/datasource/csvsource/tick_downloader_test.go
Normal file
|
@ -0,0 +1,103 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DownloadTester struct {
|
||||||
|
Exchange types.ExchangeName
|
||||||
|
Reader MakeCSVTickReader
|
||||||
|
Market MarketType
|
||||||
|
Granularity DataType
|
||||||
|
Symbol string
|
||||||
|
Path string
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
expectedCandles = []int{1440, 48, 24}
|
||||||
|
intervals = []types.Interval{types.Interval1m, types.Interval30m, types.Interval1h}
|
||||||
|
until = time.Now().Round(0)
|
||||||
|
since = until.Add(-24 * time.Hour)
|
||||||
|
)
|
||||||
|
|
||||||
|
func Test_CSV_Download(t *testing.T) {
|
||||||
|
if _, ok := os.LookupEnv("TEST_CSV_DOWNLOADER"); !ok {
|
||||||
|
t.Skip()
|
||||||
|
}
|
||||||
|
var tests = []DownloadTester{
|
||||||
|
{
|
||||||
|
Exchange: types.ExchangeBinance,
|
||||||
|
Reader: NewBinanceCSVTickReader,
|
||||||
|
Market: SPOT,
|
||||||
|
Granularity: AGGTRADES,
|
||||||
|
Symbol: "FXSUSDT",
|
||||||
|
Path: "testdata/binance/FXSUSDT",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Exchange: types.ExchangeBybit,
|
||||||
|
Reader: NewBybitCSVTickReader,
|
||||||
|
Market: FUTURES,
|
||||||
|
Granularity: AGGTRADES,
|
||||||
|
Symbol: "FXSUSDT",
|
||||||
|
Path: "testdata/bybit/FXSUSDT",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Exchange: types.ExchangeOKEx,
|
||||||
|
Reader: NewOKExCSVTickReader,
|
||||||
|
Market: SPOT,
|
||||||
|
Granularity: AGGTRADES,
|
||||||
|
Symbol: "BTCUSDT",
|
||||||
|
Path: "testdata/okex/BTCUSDT",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
err := Download(
|
||||||
|
tt.Path,
|
||||||
|
tt.Symbol,
|
||||||
|
tt.Exchange,
|
||||||
|
tt.Market,
|
||||||
|
tt.Granularity,
|
||||||
|
since,
|
||||||
|
until,
|
||||||
|
)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
klineMap, err := ReadTicksFromCSVWithDecoder(
|
||||||
|
tt.Path,
|
||||||
|
tt.Symbol,
|
||||||
|
intervals,
|
||||||
|
MakeCSVTickReader(tt.Reader),
|
||||||
|
)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
for i, interval := range intervals {
|
||||||
|
klines := klineMap[interval]
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
expectedCandles[i],
|
||||||
|
len(klines),
|
||||||
|
fmt.Sprintf("%s: %s/%s should have %d kLines",
|
||||||
|
tt.Exchange.String(),
|
||||||
|
tt.Symbol,
|
||||||
|
interval.String(),
|
||||||
|
expectedCandles[i],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
err = WriteKLines(tt.Path, tt.Symbol, klines)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.RemoveAll(tt.Path)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
}
|
51
pkg/datasource/csvsource/types.go
Normal file
51
pkg/datasource/csvsource/types.go
Normal file
|
@ -0,0 +1,51 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/c9s/bbgo/pkg/fixedpoint"
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type CsvConfig struct {
|
||||||
|
Market MarketType `json:"market"`
|
||||||
|
Granularity DataType `json:"granularity"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type CsvTick struct {
|
||||||
|
Exchange types.ExchangeName `json:"exchange"`
|
||||||
|
Market MarketType `json:"market"`
|
||||||
|
TradeID uint64 `json:"tradeID"`
|
||||||
|
Symbol string `json:"symbol"`
|
||||||
|
TickDirection string `json:"tickDirection"`
|
||||||
|
Side types.SideType `json:"side"`
|
||||||
|
IsBuyerMaker bool
|
||||||
|
Size fixedpoint.Value `json:"size"`
|
||||||
|
Price fixedpoint.Value `json:"price"`
|
||||||
|
HomeNotional fixedpoint.Value `json:"homeNotional"`
|
||||||
|
ForeignNotional fixedpoint.Value `json:"foreignNotional"`
|
||||||
|
Timestamp types.MillisecondTimestamp `json:"timestamp"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CsvTick) ToGlobalTrade() (*types.Trade, error) {
|
||||||
|
var isFutures bool
|
||||||
|
if c.Market == FUTURES {
|
||||||
|
isFutures = true
|
||||||
|
}
|
||||||
|
return &types.Trade{
|
||||||
|
ID: c.TradeID,
|
||||||
|
// OrderID: // not implemented
|
||||||
|
Exchange: c.Exchange,
|
||||||
|
Price: c.Price,
|
||||||
|
Quantity: c.Size,
|
||||||
|
QuoteQuantity: c.Price.Mul(c.Size), // todo this does not seem right use of propert.. looses info on foreign notional
|
||||||
|
Symbol: c.Symbol,
|
||||||
|
Side: c.Side,
|
||||||
|
IsBuyer: c.Side == types.SideTypeBuy,
|
||||||
|
IsMaker: c.IsBuyerMaker,
|
||||||
|
Time: types.Time(c.Timestamp),
|
||||||
|
// Fee: trade.ExecFee, // info is overwritten by stream?
|
||||||
|
// FeeCurrency: trade.FeeTokenId,
|
||||||
|
IsFutures: isFutures,
|
||||||
|
IsMargin: false,
|
||||||
|
IsIsolated: false,
|
||||||
|
}, nil
|
||||||
|
}
|
77
pkg/datasource/csvsource/write_klines.go
Normal file
77
pkg/datasource/csvsource/write_klines.go
Normal file
|
@ -0,0 +1,77 @@
|
||||||
|
package csvsource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// WriteKLines writes csv to path.
|
||||||
|
func WriteKLines(path, symbol string, klines []types.KLine) (err error) {
|
||||||
|
if len(klines) == 0 {
|
||||||
|
return fmt.Errorf("no klines to write")
|
||||||
|
}
|
||||||
|
from := klines[0].StartTime.Time()
|
||||||
|
end := klines[len(klines)-1].EndTime.Time()
|
||||||
|
to := ""
|
||||||
|
if from.AddDate(0, 0, 1).After(end) {
|
||||||
|
to = "-" + end.Format("2006-01-02")
|
||||||
|
}
|
||||||
|
|
||||||
|
path = fmt.Sprintf("%s/klines/%s",
|
||||||
|
path,
|
||||||
|
klines[0].Interval.String(),
|
||||||
|
)
|
||||||
|
|
||||||
|
fileName := fmt.Sprintf("%s/%s-%s%s.csv",
|
||||||
|
path,
|
||||||
|
symbol,
|
||||||
|
from.Format("2006-01-02"),
|
||||||
|
to,
|
||||||
|
)
|
||||||
|
|
||||||
|
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
|
||||||
|
err := os.MkdirAll(path, os.ModePerm)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("mkdir %s: %w", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Create(fileName)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "failed to open file")
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
err = file.Close()
|
||||||
|
if err != nil {
|
||||||
|
panic("failed to close file")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
w := csv.NewWriter(file)
|
||||||
|
defer w.Flush()
|
||||||
|
|
||||||
|
// Using Write
|
||||||
|
for _, kline := range klines {
|
||||||
|
row := []string{
|
||||||
|
fmt.Sprintf("%d", kline.StartTime.Unix()),
|
||||||
|
kline.Open.String(),
|
||||||
|
kline.High.String(),
|
||||||
|
kline.Low.String(),
|
||||||
|
kline.Close.String(),
|
||||||
|
kline.Volume.String(),
|
||||||
|
}
|
||||||
|
if err := w.Write(row); err != nil {
|
||||||
|
return errors.Wrap(err, "writing record to file")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -112,18 +112,6 @@ func (s Slice) Average() float64 {
|
||||||
return total / float64(len(s))
|
return total / float64(len(s))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s Slice) AverageSquared() float64 {
|
|
||||||
if len(s) == 0 {
|
|
||||||
return 0.0
|
|
||||||
}
|
|
||||||
|
|
||||||
total := 0.0
|
|
||||||
for _, value := range s {
|
|
||||||
total += math.Pow(value, 2)
|
|
||||||
}
|
|
||||||
return total / float64(len(s))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s Slice) Diff() (values Slice) {
|
func (s Slice) Diff() (values Slice) {
|
||||||
for i, v := range s {
|
for i, v := range s {
|
||||||
if i == 0 {
|
if i == 0 {
|
||||||
|
|
|
@ -115,10 +115,10 @@ func convertSubscriptions(ss []types.Subscription) ([]WebSocketCommand, error) {
|
||||||
switch s.Channel {
|
switch s.Channel {
|
||||||
case types.BookChannel:
|
case types.BookChannel:
|
||||||
// see https://docs.kucoin.com/#level-2-market-data
|
// see https://docs.kucoin.com/#level-2-market-data
|
||||||
subscribeTopic = "/market/level2" + ":" + toLocalSymbol(s.Symbol)
|
subscribeTopic = "/market/level2" + ":" + ToLocalSymbol(s.Symbol)
|
||||||
|
|
||||||
case types.KLineChannel:
|
case types.KLineChannel:
|
||||||
subscribeTopic = "/market/candles" + ":" + toLocalSymbol(s.Symbol) + "_" + toLocalInterval(types.Interval(s.Options.Interval))
|
subscribeTopic = "/market/candles" + ":" + ToLocalSymbol(s.Symbol) + "_" + toLocalInterval(types.Interval(s.Options.Interval))
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("websocket channel %s is not supported by kucoin", s.Channel)
|
return nil, fmt.Errorf("websocket channel %s is not supported by kucoin", s.Channel)
|
||||||
|
|
|
@ -165,7 +165,7 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type
|
||||||
}
|
}
|
||||||
|
|
||||||
req := e.client.MarketDataService.NewGetKLinesRequest()
|
req := e.client.MarketDataService.NewGetKLinesRequest()
|
||||||
req.Symbol(toLocalSymbol(symbol))
|
req.Symbol(ToLocalSymbol(symbol))
|
||||||
req.Interval(toLocalInterval(interval))
|
req.Interval(toLocalInterval(interval))
|
||||||
if options.StartTime != nil {
|
if options.StartTime != nil {
|
||||||
req.StartAt(*options.StartTime)
|
req.StartAt(*options.StartTime)
|
||||||
|
@ -208,7 +208,7 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type
|
||||||
|
|
||||||
func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (createdOrder *types.Order, err error) {
|
func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (createdOrder *types.Order, err error) {
|
||||||
req := e.client.TradeService.NewPlaceOrderRequest()
|
req := e.client.TradeService.NewPlaceOrderRequest()
|
||||||
req.Symbol(toLocalSymbol(order.Symbol))
|
req.Symbol(ToLocalSymbol(order.Symbol))
|
||||||
req.Side(toLocalSide(order.Side))
|
req.Side(toLocalSide(order.Side))
|
||||||
|
|
||||||
if order.ClientOrderID != "" {
|
if order.ClientOrderID != "" {
|
||||||
|
@ -298,7 +298,7 @@ You will not be able to query for cancelled orders that have happened more than
|
||||||
*/
|
*/
|
||||||
func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders []types.Order, err error) {
|
func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders []types.Order, err error) {
|
||||||
req := e.client.TradeService.NewListOrdersRequest()
|
req := e.client.TradeService.NewListOrdersRequest()
|
||||||
req.Symbol(toLocalSymbol(symbol))
|
req.Symbol(ToLocalSymbol(symbol))
|
||||||
req.Status("active")
|
req.Status("active")
|
||||||
orderList, err := req.Do(ctx)
|
orderList, err := req.Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -316,7 +316,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [
|
||||||
|
|
||||||
func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) {
|
func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) {
|
||||||
req := e.client.TradeService.NewListOrdersRequest()
|
req := e.client.TradeService.NewListOrdersRequest()
|
||||||
req.Symbol(toLocalSymbol(symbol))
|
req.Symbol(ToLocalSymbol(symbol))
|
||||||
req.Status("done")
|
req.Status("done")
|
||||||
req.StartAt(since)
|
req.StartAt(since)
|
||||||
|
|
||||||
|
@ -350,7 +350,7 @@ var launchDate = time.Date(2017, 9, 0, 0, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) {
|
func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) {
|
||||||
req := e.client.TradeService.NewGetFillsRequest()
|
req := e.client.TradeService.NewGetFillsRequest()
|
||||||
req.Symbol(toLocalSymbol(symbol))
|
req.Symbol(ToLocalSymbol(symbol))
|
||||||
|
|
||||||
// we always sync trades in the ascending order, and kucoin does not support last trade ID query
|
// we always sync trades in the ascending order, and kucoin does not support last trade ID query
|
||||||
// hence we need to set the start time here
|
// hence we need to set the start time here
|
||||||
|
@ -422,7 +422,7 @@ func (e *Exchange) NewStream() types.Stream {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Exchange) QueryDepth(ctx context.Context, symbol string) (types.SliceOrderBook, int64, error) {
|
func (e *Exchange) QueryDepth(ctx context.Context, symbol string) (types.SliceOrderBook, int64, error) {
|
||||||
orderBook, err := e.client.MarketDataService.GetOrderBook(toLocalSymbol(symbol), 100)
|
orderBook, err := e.client.MarketDataService.GetOrderBook(ToLocalSymbol(symbol), 100)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return types.SliceOrderBook{}, 0, err
|
return types.SliceOrderBook{}, 0, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,13 +17,13 @@ import (
|
||||||
var packageTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
|
var packageTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
|
||||||
package kucoin
|
package kucoin
|
||||||
|
|
||||||
var symbolMap = map[string]string{
|
var SymbolMap = map[string]string{
|
||||||
{{- range $k, $v := . }}
|
{{- range $k, $v := . }}
|
||||||
{{ printf "%q" $k }}: {{ printf "%q" $v }},
|
{{ printf "%q" $k }}: {{ printf "%q" $v }},
|
||||||
{{- end }}
|
{{- end }}
|
||||||
}
|
}
|
||||||
|
|
||||||
func toLocalSymbol(symbol string) string {
|
func ToLocalSymbol(symbol string) string {
|
||||||
s, ok := symbolMap[symbol]
|
s, ok := symbolMap[symbol]
|
||||||
if ok {
|
if ok {
|
||||||
return s
|
return s
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
// Code generated by go generate; DO NOT EDIT.
|
// Code generated by go generate; DO NOT EDIT.
|
||||||
package kucoin
|
package kucoin
|
||||||
|
|
||||||
var symbolMap = map[string]string{
|
var SymbolMap = map[string]string{
|
||||||
"1EARTHUSDT": "1EARTH-USDT",
|
"1EARTHUSDT": "1EARTH-USDT",
|
||||||
"1INCHUSDT": "1INCH-USDT",
|
"1INCHUSDT": "1INCH-USDT",
|
||||||
"2CRZBTC": "2CRZ-BTC",
|
"2CRZBTC": "2CRZ-BTC",
|
||||||
|
@ -1107,8 +1107,8 @@ var symbolMap = map[string]string{
|
||||||
"ZRXETH": "ZRX-ETH",
|
"ZRXETH": "ZRX-ETH",
|
||||||
}
|
}
|
||||||
|
|
||||||
func toLocalSymbol(symbol string) string {
|
func ToLocalSymbol(symbol string) string {
|
||||||
s, ok := symbolMap[symbol]
|
s, ok := SymbolMap[symbol]
|
||||||
if ok {
|
if ok {
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
10
pkg/indicator/v2/testdata/BTCUSDT-1m-2022-05-06.csv
vendored
Normal file
10
pkg/indicator/v2/testdata/BTCUSDT-1m-2022-05-06.csv
vendored
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
1651795200000,36533.70,36540.00,36501.00,36505.20,264.779,1651795259999,9670700.33840,3057,71.011,2593768.86330,0
|
||||||
|
1651795260000,36506.30,36523.10,36492.30,36522.70,180.741,1651795319999,6598288.01340,2214,70.811,2585241.60220,0
|
||||||
|
1651795320000,36522.70,36559.10,36518.90,36549.60,280.910,1651795379999,10263878.29160,2898,155.711,5689249.26850,0
|
||||||
|
1651795380000,36549.90,36550.00,36490.00,36534.40,235.291,1651795439999,8591157.31110,2690,78.925,2881502.53680,0
|
||||||
|
1651795440000,36534.40,36577.50,36534.40,36574.80,218.490,1651795499999,7988553.23400,2184,133.092,4866125.50710,0
|
||||||
|
1651795500000,36574.90,36679.30,36561.40,36611.60,1180.452,1651795559999,43233700.14416,8720,852.525,31228536.48026,0
|
||||||
|
1651795560000,36611.60,36614.60,36588.20,36612.70,252.435,1651795619999,9240546.27360,2494,104.381,3821126.58030,0
|
||||||
|
1651795620000,36612.80,36647.10,36586.10,36594.50,361.987,1651795679999,13254573.37270,3565,220.110,8060195.17170,0
|
||||||
|
1651795680000,36594.60,36598.10,36543.00,36566.60,236.064,1651795739999,8631772.05423,2650,66.766,2441168.29810,0
|
||||||
|
1651795740000,36565.90,36565.90,36525.90,36530.80,129.389,1651795799999,4728306.04240,1697,45.836,1674990.33390,0
|
|
194
pkg/indicator/v2/volume_profile.go
Normal file
194
pkg/indicator/v2/volume_profile.go
Normal file
|
@ -0,0 +1,194 @@
|
||||||
|
package indicatorv2
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
|
||||||
|
"golang.org/x/exp/slices"
|
||||||
|
"gonum.org/v1/gonum/floats"
|
||||||
|
"gonum.org/v1/gonum/stat"
|
||||||
|
|
||||||
|
bbgofloats "github.com/c9s/bbgo/pkg/datatype/floats"
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultValueAreaPercentage is the percentage of the total volume used to calculate the value area.
|
||||||
|
const DefaultValueAreaPercentage = 0.68
|
||||||
|
|
||||||
|
type VolumeProfileStream struct {
|
||||||
|
*types.Float64Series
|
||||||
|
VP VolumeProfile
|
||||||
|
window int
|
||||||
|
}
|
||||||
|
|
||||||
|
// VolumeProfile is a histogram of market price and volume.
|
||||||
|
// Intent is to show the price points with most volume during a period.
|
||||||
|
// The profile gives key features such as:
|
||||||
|
//
|
||||||
|
// Point of control (POC)
|
||||||
|
//
|
||||||
|
// Value area high (VAH)
|
||||||
|
//
|
||||||
|
// Value area low (VAL)
|
||||||
|
//
|
||||||
|
// Session High/Low
|
||||||
|
type VolumeProfile struct {
|
||||||
|
|
||||||
|
// Bins is the histogram bins.
|
||||||
|
Bins []float64
|
||||||
|
|
||||||
|
// Hist is the histogram values.
|
||||||
|
Hist []float64
|
||||||
|
|
||||||
|
// POC is the point of control.
|
||||||
|
POC float64
|
||||||
|
|
||||||
|
// VAH is the value area high.
|
||||||
|
VAH float64
|
||||||
|
|
||||||
|
// VAL is the value area low.
|
||||||
|
VAL float64
|
||||||
|
|
||||||
|
// High is the highest price in the profile.
|
||||||
|
High float64
|
||||||
|
|
||||||
|
// Low is the lowest price in the profile.
|
||||||
|
Low float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// VolumeLevel is a price and volume pair used to build a volume profile.
|
||||||
|
type VolumeLevel struct {
|
||||||
|
|
||||||
|
// Price is the market price, typically the high/low average of the kline.
|
||||||
|
Price float64
|
||||||
|
|
||||||
|
// Volume is the total buy and sell volume at the price.
|
||||||
|
Volume float64
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewVolumeProfile(source KLineSubscription, window int) *VolumeProfileStream {
|
||||||
|
prices := HLC3(source)
|
||||||
|
volumes := Volumes(source)
|
||||||
|
|
||||||
|
s := &VolumeProfileStream{
|
||||||
|
Float64Series: types.NewFloat64Series(),
|
||||||
|
window: window,
|
||||||
|
}
|
||||||
|
|
||||||
|
source.AddSubscriber(func(v types.KLine) {
|
||||||
|
if source.Length() < window {
|
||||||
|
s.PushAndEmit(0)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var nBins = 10
|
||||||
|
// nBins = int(math.Floor((prices.Slice.Max()-prices.Slice.Min())/binWidth)) + 1
|
||||||
|
s.VP.High = prices.Slice.Max()
|
||||||
|
s.VP.Low = prices.Slice.Min()
|
||||||
|
sortedPrices, sortedVolumes := buildVolumeLevel(prices.Slice, volumes.Slice)
|
||||||
|
s.VP.Bins = make([]float64, nBins)
|
||||||
|
s.VP.Bins = floats.Span(s.VP.Bins, s.VP.Low, s.VP.High+1)
|
||||||
|
s.VP.Hist = stat.Histogram(nil, s.VP.Bins, sortedPrices, sortedVolumes)
|
||||||
|
|
||||||
|
pocIdx := floats.MaxIdx(s.VP.Hist)
|
||||||
|
s.VP.POC = midBin(s.VP.Bins, pocIdx)
|
||||||
|
|
||||||
|
// TODO the results are of by small difference whereas it is expected they work the same
|
||||||
|
// vaTotalVol := volumes.Sum() * DefaultValueAreaPercentage
|
||||||
|
// Calculate Value Area with POC as the centre point\
|
||||||
|
vaTotalVol := floats.Sum(volumes.Slice) * DefaultValueAreaPercentage
|
||||||
|
|
||||||
|
vaCumVol := s.VP.Hist[pocIdx]
|
||||||
|
var vahVol, valVol float64
|
||||||
|
vahIdx, valIdx := pocIdx+1, pocIdx-1
|
||||||
|
stepVAH, stepVAL := true, true
|
||||||
|
|
||||||
|
for (vaCumVol <= vaTotalVol) &&
|
||||||
|
(vahIdx <= len(s.VP.Hist)-1 && valIdx >= 0) {
|
||||||
|
|
||||||
|
if stepVAH {
|
||||||
|
vahVol = 0
|
||||||
|
for vahVol == 0 && vahIdx+1 < len(s.VP.Hist)-1 {
|
||||||
|
vahVol = s.VP.Hist[vahIdx] + s.VP.Hist[vahIdx+1]
|
||||||
|
vahIdx += 2
|
||||||
|
}
|
||||||
|
stepVAH = false
|
||||||
|
}
|
||||||
|
|
||||||
|
if stepVAL {
|
||||||
|
valVol = 0
|
||||||
|
for valVol == 0 && valIdx-1 >= 0 {
|
||||||
|
valVol = s.VP.Hist[valIdx] + s.VP.Hist[valIdx-1]
|
||||||
|
valIdx -= 2
|
||||||
|
}
|
||||||
|
stepVAL = false
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case vahVol > valVol:
|
||||||
|
vaCumVol += vahVol
|
||||||
|
stepVAH, stepVAL = true, false
|
||||||
|
case vahVol < valVol:
|
||||||
|
vaCumVol += valVol
|
||||||
|
stepVAH, stepVAL = false, true
|
||||||
|
case vahVol == valVol:
|
||||||
|
vaCumVol += valVol + vahVol
|
||||||
|
stepVAH, stepVAL = true, true
|
||||||
|
}
|
||||||
|
|
||||||
|
if vahIdx >= len(s.VP.Hist)-1 {
|
||||||
|
stepVAH = false
|
||||||
|
}
|
||||||
|
|
||||||
|
if valIdx <= 0 {
|
||||||
|
stepVAL = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
s.VP.VAH = midBin(s.VP.Bins, vahIdx)
|
||||||
|
s.VP.VAL = midBin(s.VP.Bins, valIdx)
|
||||||
|
|
||||||
|
})
|
||||||
|
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *VolumeProfileStream) Truncate() {
|
||||||
|
s.Slice = s.Slice.Truncate(5000)
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildVolumeLevel(p, v bbgofloats.Slice) (sortedp, sortedv bbgofloats.Slice) {
|
||||||
|
var levels []VolumeLevel
|
||||||
|
for i := range p {
|
||||||
|
levels = append(levels, VolumeLevel{
|
||||||
|
Price: p[i],
|
||||||
|
Volume: v[i],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
slices.SortStableFunc(levels, func(i, j VolumeLevel) bool {
|
||||||
|
return i.Price < j.Price
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, v := range levels {
|
||||||
|
sortedp.Append(v.Price)
|
||||||
|
sortedv.Append(v.Volume)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func midBin(bins []float64, idx int) float64 {
|
||||||
|
|
||||||
|
if len(bins) == 0 {
|
||||||
|
return math.NaN()
|
||||||
|
}
|
||||||
|
|
||||||
|
if idx >= len(bins)-1 {
|
||||||
|
return bins[len(bins)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
if idx < 0 {
|
||||||
|
return bins[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
return stat.Mean([]float64{bins[idx], bins[idx+1]}, nil)
|
||||||
|
}
|
37
pkg/indicator/v2/volume_profile_test.go
Normal file
37
pkg/indicator/v2/volume_profile_test.go
Normal file
|
@ -0,0 +1,37 @@
|
||||||
|
package indicatorv2
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/datasource/csvsource"
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestVolumeProfile(t *testing.T) {
|
||||||
|
file, _ := os.Open(path.Join("testdata", "BTCUSDT-1m-2022-05-06.csv"))
|
||||||
|
defer func() {
|
||||||
|
assert.NoError(t, file.Close())
|
||||||
|
}()
|
||||||
|
|
||||||
|
candles, err := csvsource.NewCSVKLineReader(csv.NewReader(file)).ReadAll(time.Minute)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
stream := &types.StandardStream{}
|
||||||
|
kLines := KLines(stream, "", "")
|
||||||
|
ind := NewVolumeProfile(kLines, 10)
|
||||||
|
|
||||||
|
for _, candle := range candles {
|
||||||
|
stream.EmitKLineClosed(candle)
|
||||||
|
}
|
||||||
|
assert.InDelta(t, 36512.7, ind.VP.Low, 0.01, "VP.LOW")
|
||||||
|
assert.InDelta(t, 36512.7, ind.VP.VAL, 0.01, "VP.VAL")
|
||||||
|
assert.InDelta(t, 36518.574, ind.VP.POC, 0.01, "VP.POC")
|
||||||
|
assert.InDelta(t, 36530.322, ind.VP.VAH, 0.01, "VP.VAH")
|
||||||
|
assert.InDelta(t, 36617.433, ind.VP.High, 0.01, "VP.HIGH")
|
||||||
|
}
|
163
pkg/service/backtest_csv.go
Normal file
163
pkg/service/backtest_csv.go
Normal file
|
@ -0,0 +1,163 @@
|
||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/c9s/bbgo/pkg/datasource/csvsource"
|
||||||
|
exchange2 "github.com/c9s/bbgo/pkg/exchange"
|
||||||
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type BacktestServiceCSV struct {
|
||||||
|
kLines map[types.Interval][]types.KLine
|
||||||
|
path string
|
||||||
|
market csvsource.MarketType
|
||||||
|
granularity csvsource.DataType
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewBacktestServiceCSV(
|
||||||
|
path string,
|
||||||
|
market csvsource.MarketType,
|
||||||
|
granularity csvsource.DataType,
|
||||||
|
) BackTestable {
|
||||||
|
return &BacktestServiceCSV{
|
||||||
|
kLines: make(map[types.Interval][]types.KLine),
|
||||||
|
path: path,
|
||||||
|
market: market,
|
||||||
|
granularity: granularity,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *BacktestServiceCSV) Verify(sourceExchange types.Exchange, symbols []string, startTime time.Time, endTime time.Time) error {
|
||||||
|
// TODO: use isFutures here
|
||||||
|
_, _, isIsolated, isolatedSymbol := exchange2.GetSessionAttributes(sourceExchange)
|
||||||
|
// override symbol if isolatedSymbol is not empty
|
||||||
|
if isIsolated && len(isolatedSymbol) > 0 {
|
||||||
|
err := csvsource.Download(
|
||||||
|
s.path,
|
||||||
|
isolatedSymbol,
|
||||||
|
sourceExchange.Name(),
|
||||||
|
s.market,
|
||||||
|
s.granularity,
|
||||||
|
startTime,
|
||||||
|
endTime,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Errorf("downloading csv data: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *BacktestServiceCSV) Sync(ctx context.Context, exchange types.Exchange, symbol string, intervals []types.Interval, startTime, endTime time.Time) error {
|
||||||
|
|
||||||
|
log.Infof("starting fresh csv sync %s %s: %s <=> %s", exchange.Name(), symbol, startTime, endTime)
|
||||||
|
|
||||||
|
path := fmt.Sprintf("%s/%s/%s", s.path, exchange.Name().String(), symbol)
|
||||||
|
|
||||||
|
var reader csvsource.MakeCSVTickReader
|
||||||
|
|
||||||
|
switch exchange.Name() {
|
||||||
|
case types.ExchangeBinance:
|
||||||
|
reader = csvsource.NewBinanceCSVTickReader
|
||||||
|
case types.ExchangeBybit:
|
||||||
|
reader = csvsource.NewBybitCSVTickReader
|
||||||
|
case types.ExchangeOKEx:
|
||||||
|
reader = csvsource.NewOKExCSVTickReader
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s not supported yet.. care to fix it? PR's welcome ;)", exchange.Name().String())
|
||||||
|
}
|
||||||
|
|
||||||
|
kLineMap, err := csvsource.ReadTicksFromCSVWithDecoder(
|
||||||
|
path,
|
||||||
|
symbol,
|
||||||
|
intervals,
|
||||||
|
csvsource.MakeCSVTickReader(reader),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Errorf("reading csv data: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.kLines = kLineMap
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueryKLine queries the klines from the database
|
||||||
|
func (s *BacktestServiceCSV) QueryKLine(ex types.ExchangeName, symbol string, interval types.Interval, orderBy string, limit int) (*types.KLine, error) {
|
||||||
|
log.Infof("querying last kline exchange = %s AND symbol = %s AND interval = %s", ex, symbol, interval)
|
||||||
|
if _, ok := s.kLines[interval]; !ok || len(s.kLines[interval]) == 0 {
|
||||||
|
return nil, errors.New("interval not initialized")
|
||||||
|
}
|
||||||
|
return &s.kLines[interval][len(s.kLines[interval])-1], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueryKLinesForward is used for querying klines to back-testing
|
||||||
|
func (s *BacktestServiceCSV) QueryKLinesForward(exchange types.ExchangeName, symbol string, interval types.Interval, startTime time.Time, limit int) ([]types.KLine, error) {
|
||||||
|
// Sample implementation (modify as needed):
|
||||||
|
var result []types.KLine
|
||||||
|
|
||||||
|
// Access klines data based on exchange, symbol, and interval
|
||||||
|
exchangeKLines, ok := s.kLines[interval]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("no kLines for specified interval %s", interval.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter klines based on startTime and limit
|
||||||
|
for _, kline := range exchangeKLines {
|
||||||
|
if kline.StartTime.After(startTime) {
|
||||||
|
result = append(result, kline)
|
||||||
|
if len(result) >= limit {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *BacktestServiceCSV) QueryKLinesBackward(exchange types.ExchangeName, symbol string, interval types.Interval, endTime time.Time, limit int) ([]types.KLine, error) {
|
||||||
|
var result []types.KLine
|
||||||
|
|
||||||
|
// Access klines data based on interval
|
||||||
|
exchangeKLines, ok := s.kLines[interval]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("no kLines for specified interval %s", interval.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reverse iteration through klines and filter based on endTime and limit
|
||||||
|
for i := len(exchangeKLines) - 1; i >= 0; i-- {
|
||||||
|
kline := exchangeKLines[i]
|
||||||
|
|
||||||
|
if kline.StartTime.Before(endTime) {
|
||||||
|
result = append(result, kline)
|
||||||
|
if len(result) >= limit {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *BacktestServiceCSV) QueryKLinesCh(since, until time.Time, exchange types.Exchange, symbols []string, intervals []types.Interval) (chan types.KLine, chan error) {
|
||||||
|
if len(symbols) == 0 {
|
||||||
|
return returnError(errors.Errorf("symbols is empty when querying kline, please check your strategy setting. "))
|
||||||
|
}
|
||||||
|
|
||||||
|
ch := make(chan types.KLine, len(s.kLines))
|
||||||
|
go func() {
|
||||||
|
defer close(ch)
|
||||||
|
for _, kline := range s.kLines[intervals[0]] {
|
||||||
|
ch <- kline
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return ch, nil
|
||||||
|
}
|
|
@ -18,14 +18,26 @@ import (
|
||||||
"github.com/c9s/bbgo/pkg/types"
|
"github.com/c9s/bbgo/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type BackTestable interface {
|
||||||
|
Verify(sourceExchange types.Exchange, symbols []string, startTime time.Time, endTime time.Time) error
|
||||||
|
Sync(ctx context.Context, ex types.Exchange, symbol string, intervals []types.Interval, since, until time.Time) error
|
||||||
|
QueryKLine(ex types.ExchangeName, symbol string, interval types.Interval, orderBy string, limit int) (*types.KLine, error)
|
||||||
|
QueryKLinesForward(exchange types.ExchangeName, symbol string, interval types.Interval, startTime time.Time, limit int) ([]types.KLine, error)
|
||||||
|
QueryKLinesBackward(exchange types.ExchangeName, symbol string, interval types.Interval, endTime time.Time, limit int) ([]types.KLine, error)
|
||||||
|
QueryKLinesCh(since, until time.Time, exchange types.Exchange, symbols []string, intervals []types.Interval) (chan types.KLine, chan error)
|
||||||
|
}
|
||||||
|
|
||||||
type BacktestService struct {
|
type BacktestService struct {
|
||||||
DB *sqlx.DB
|
DB *sqlx.DB
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *BacktestService) SyncKLineByInterval(
|
func NewBacktestService(db *sqlx.DB) *BacktestService {
|
||||||
ctx context.Context, exchange types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time,
|
return &BacktestService{DB: db}
|
||||||
) error {
|
}
|
||||||
|
|
||||||
|
func (s *BacktestService) syncKLineByInterval(ctx context.Context, exchange types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time) error {
|
||||||
_, isFutures, isIsolated, isolatedSymbol := exchange2.GetSessionAttributes(exchange)
|
_, isFutures, isIsolated, isolatedSymbol := exchange2.GetSessionAttributes(exchange)
|
||||||
|
log.Infof("synchronizing %s klines with interval %s: %s <=> %s", exchange.Name(), interval, startTime, endTime)
|
||||||
|
|
||||||
// override symbol if isolatedSymbol is not empty
|
// override symbol if isolatedSymbol is not empty
|
||||||
if isIsolated && len(isolatedSymbol) > 0 {
|
if isIsolated && len(isolatedSymbol) > 0 {
|
||||||
|
@ -101,7 +113,7 @@ func (s *BacktestService) Verify(sourceExchange types.Exchange, symbols []string
|
||||||
for interval := range types.SupportedIntervals {
|
for interval := range types.SupportedIntervals {
|
||||||
log.Infof("verifying %s %s backtesting data: %s to %s...", symbol, interval, startTime, endTime)
|
log.Infof("verifying %s %s backtesting data: %s to %s...", symbol, interval, startTime, endTime)
|
||||||
|
|
||||||
timeRanges, err := s.FindMissingTimeRanges(context.Background(), sourceExchange, symbol, interval,
|
timeRanges, err := s.findMissingTimeRanges(context.Background(), sourceExchange, symbol, interval,
|
||||||
startTime, endTime)
|
startTime, endTime)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -129,13 +141,11 @@ func (s *BacktestService) Verify(sourceExchange types.Exchange, symbols []string
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *BacktestService) SyncFresh(
|
func (s *BacktestService) SyncFresh(ctx context.Context, exchange types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time) error {
|
||||||
ctx context.Context, exchange types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time,
|
|
||||||
) error {
|
|
||||||
log.Infof("starting fresh sync %s %s %s: %s <=> %s", exchange.Name(), symbol, interval, startTime, endTime)
|
log.Infof("starting fresh sync %s %s %s: %s <=> %s", exchange.Name(), symbol, interval, startTime, endTime)
|
||||||
startTime = startTime.Truncate(time.Minute).Add(-2 * time.Second)
|
startTime = startTime.Truncate(time.Minute).Add(-2 * time.Second)
|
||||||
endTime = endTime.Truncate(time.Minute).Add(2 * time.Second)
|
endTime = endTime.Truncate(time.Minute).Add(2 * time.Second)
|
||||||
return s.SyncKLineByInterval(ctx, exchange, symbol, interval, startTime, endTime)
|
return s.syncKLineByInterval(ctx, exchange, symbol, interval, startTime, endTime)
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueryKLine queries the klines from the database
|
// QueryKLine queries the klines from the database
|
||||||
|
@ -373,20 +383,28 @@ func (t *TimeRange) String() string {
|
||||||
return t.Start.String() + " ~ " + t.End.String()
|
return t.Start.String() + " ~ " + t.End.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *BacktestService) Sync(
|
func (s *BacktestService) Sync(ctx context.Context, ex types.Exchange, symbol string, intervals []types.Interval, since, until time.Time) error {
|
||||||
ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time,
|
for _, interval := range intervals {
|
||||||
) error {
|
t1, t2, err := s.queryExistingDataRange(ctx, ex, symbol, interval, since, until)
|
||||||
t1, t2, err := s.QueryExistingDataRange(ctx, ex, symbol, interval, since, until)
|
if err != nil && err != sql.ErrNoRows {
|
||||||
if err != nil && err != sql.ErrNoRows {
|
return err
|
||||||
return err
|
}
|
||||||
|
|
||||||
|
if err == sql.ErrNoRows || t1 == nil || t2 == nil {
|
||||||
|
// fallback to fresh sync
|
||||||
|
err := s.syncFresh(ctx, ex, symbol, interval, since, until)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
err := s.syncPartial(ctx, ex, symbol, interval, since, until)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err == sql.ErrNoRows || t1 == nil || t2 == nil {
|
return nil
|
||||||
// fallback to fresh sync
|
|
||||||
return s.SyncFresh(ctx, ex, symbol, interval, since, until)
|
|
||||||
}
|
|
||||||
|
|
||||||
return s.SyncPartial(ctx, ex, symbol, interval, since, until)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// SyncPartial
|
// SyncPartial
|
||||||
|
@ -394,22 +412,20 @@ func (s *BacktestService) Sync(
|
||||||
// scan if there is a missing part
|
// scan if there is a missing part
|
||||||
// create a time range slice []TimeRange
|
// create a time range slice []TimeRange
|
||||||
// iterate the []TimeRange slice to sync data.
|
// iterate the []TimeRange slice to sync data.
|
||||||
func (s *BacktestService) SyncPartial(
|
func (s *BacktestService) syncPartial(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time) error {
|
||||||
ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time,
|
|
||||||
) error {
|
|
||||||
log.Infof("starting partial sync %s %s %s: %s <=> %s", ex.Name(), symbol, interval, since, until)
|
log.Infof("starting partial sync %s %s %s: %s <=> %s", ex.Name(), symbol, interval, since, until)
|
||||||
|
|
||||||
t1, t2, err := s.QueryExistingDataRange(ctx, ex, symbol, interval, since, until)
|
t1, t2, err := s.queryExistingDataRange(ctx, ex, symbol, interval, since, until)
|
||||||
if err != nil && err != sql.ErrNoRows {
|
if err != nil && err != sql.ErrNoRows {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err == sql.ErrNoRows || t1 == nil || t2 == nil {
|
if err == sql.ErrNoRows || t1 == nil || t2 == nil {
|
||||||
// fallback to fresh sync
|
// fallback to fresh sync
|
||||||
return s.SyncFresh(ctx, ex, symbol, interval, since, until)
|
return s.syncFresh(ctx, ex, symbol, interval, since, until)
|
||||||
}
|
}
|
||||||
|
|
||||||
timeRanges, err := s.FindMissingTimeRanges(ctx, ex, symbol, interval, t1.Time(), t2.Time())
|
timeRanges, err := s.findMissingTimeRanges(ctx, ex, symbol, interval, t1.Time(), t2.Time())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -436,7 +452,7 @@ func (s *BacktestService) SyncPartial(
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, timeRange := range timeRanges {
|
for _, timeRange := range timeRanges {
|
||||||
err = s.SyncKLineByInterval(ctx, ex, symbol, interval, timeRange.Start.Add(time.Second), timeRange.End.Add(-time.Second))
|
err = s.syncKLineByInterval(ctx, ex, symbol, interval, timeRange.Start.Add(time.Second), timeRange.End.Add(-time.Second))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -447,9 +463,7 @@ func (s *BacktestService) SyncPartial(
|
||||||
|
|
||||||
// FindMissingTimeRanges returns the missing time ranges, the start/end time represents the existing data time points.
|
// FindMissingTimeRanges returns the missing time ranges, the start/end time represents the existing data time points.
|
||||||
// So when sending kline query to the exchange API, we need to add one second to the start time and minus one second to the end time.
|
// So when sending kline query to the exchange API, we need to add one second to the start time and minus one second to the end time.
|
||||||
func (s *BacktestService) FindMissingTimeRanges(
|
func (s *BacktestService) findMissingTimeRanges(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time) ([]TimeRange, error) {
|
||||||
ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time,
|
|
||||||
) ([]TimeRange, error) {
|
|
||||||
query := s.SelectKLineTimePoints(ex, symbol, interval, since, until)
|
query := s.SelectKLineTimePoints(ex, symbol, interval, since, until)
|
||||||
sql, args, err := query.ToSql()
|
sql, args, err := query.ToSql()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -492,9 +506,7 @@ func (s *BacktestService) FindMissingTimeRanges(
|
||||||
return timeRanges, nil
|
return timeRanges, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *BacktestService) QueryExistingDataRange(
|
func (s *BacktestService) queryExistingDataRange(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, tArgs ...time.Time) (start, end *types.Time, err error) {
|
||||||
ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, tArgs ...time.Time,
|
|
||||||
) (start, end *types.Time, err error) {
|
|
||||||
sel := s.SelectKLineTimeRange(ex, symbol, interval, tArgs...)
|
sel := s.SelectKLineTimeRange(ex, symbol, interval, tArgs...)
|
||||||
sql, args, err := sel.ToSql()
|
sql, args, err := sel.ToSql()
|
||||||
if err != nil {
|
if err != nil {
|
|
@ -40,7 +40,7 @@ func TestBacktestService_FindMissingTimeRanges_EmptyData(t *testing.T) {
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
startTime1 := now.AddDate(0, 0, -7).Truncate(time.Hour)
|
startTime1 := now.AddDate(0, 0, -7).Truncate(time.Hour)
|
||||||
endTime1 := now.AddDate(0, 0, -6).Truncate(time.Hour)
|
endTime1 := now.AddDate(0, 0, -6).Truncate(time.Hour)
|
||||||
timeRanges, err := service.FindMissingTimeRanges(ctx, ex, symbol, types.Interval1h, startTime1, endTime1)
|
timeRanges, err := service.findMissingTimeRanges(ctx, ex, symbol, types.Interval1h, startTime1, endTime1)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.NotEmpty(t, timeRanges)
|
assert.NotEmpty(t, timeRanges)
|
||||||
}
|
}
|
||||||
|
@ -70,7 +70,7 @@ func TestBacktestService_QueryExistingDataRange(t *testing.T) {
|
||||||
startTime1 := now.AddDate(0, 0, -7).Truncate(time.Hour)
|
startTime1 := now.AddDate(0, 0, -7).Truncate(time.Hour)
|
||||||
endTime1 := now.AddDate(0, 0, -6).Truncate(time.Hour)
|
endTime1 := now.AddDate(0, 0, -6).Truncate(time.Hour)
|
||||||
// empty range
|
// empty range
|
||||||
t1, t2, err := service.QueryExistingDataRange(ctx, ex, symbol, types.Interval1h, startTime1, endTime1)
|
t1, t2, err := service.queryExistingDataRange(ctx, ex, symbol, types.Interval1h, startTime1, endTime1)
|
||||||
assert.Error(t, sql.ErrNoRows, err)
|
assert.Error(t, sql.ErrNoRows, err)
|
||||||
assert.Nil(t, t1)
|
assert.Nil(t, t1)
|
||||||
assert.Nil(t, t2)
|
assert.Nil(t, t2)
|
||||||
|
@ -105,22 +105,22 @@ func TestBacktestService_SyncPartial(t *testing.T) {
|
||||||
endTime2 := now.AddDate(0, 0, -4).Truncate(time.Hour)
|
endTime2 := now.AddDate(0, 0, -4).Truncate(time.Hour)
|
||||||
|
|
||||||
// kline query is exclusive
|
// kline query is exclusive
|
||||||
err = service.SyncKLineByInterval(ctx, ex, symbol, types.Interval1h, startTime1.Add(-time.Second), endTime1.Add(time.Second))
|
err = service.syncKLineByInterval(ctx, ex, symbol, types.Interval1h, startTime1.Add(-time.Second), endTime1.Add(time.Second))
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
err = service.SyncKLineByInterval(ctx, ex, symbol, types.Interval1h, startTime2.Add(-time.Second), endTime2.Add(time.Second))
|
err = service.syncKLineByInterval(ctx, ex, symbol, types.Interval1h, startTime2.Add(-time.Second), endTime2.Add(time.Second))
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
timeRanges, err := service.FindMissingTimeRanges(ctx, ex, symbol, types.Interval1h, startTime1, endTime2)
|
timeRanges, err := service.findMissingTimeRanges(ctx, ex, symbol, types.Interval1h, startTime1, endTime2)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.NotEmpty(t, timeRanges)
|
assert.NotEmpty(t, timeRanges)
|
||||||
assert.Len(t, timeRanges, 1)
|
assert.Len(t, timeRanges, 1)
|
||||||
|
|
||||||
t.Run("fill missing time ranges", func(t *testing.T) {
|
t.Run("fill missing time ranges", func(t *testing.T) {
|
||||||
err = service.SyncPartial(ctx, ex, symbol, types.Interval1h, startTime1, endTime2)
|
err = service.syncPartial(ctx, ex, symbol, types.Interval1h, startTime1, endTime2)
|
||||||
assert.NoError(t, err, "sync partial should not return error")
|
assert.NoError(t, err, "sync partial should not return error")
|
||||||
|
|
||||||
timeRanges2, err := service.FindMissingTimeRanges(ctx, ex, symbol, types.Interval1h, startTime1, endTime2)
|
timeRanges2, err := service.findMissingTimeRanges(ctx, ex, symbol, types.Interval1h, startTime1, endTime2)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.Empty(t, timeRanges2)
|
assert.Empty(t, timeRanges2)
|
||||||
})
|
})
|
||||||
|
@ -155,19 +155,19 @@ func TestBacktestService_FindMissingTimeRanges(t *testing.T) {
|
||||||
endTime2 := now.AddDate(0, 0, -3).Truncate(time.Hour)
|
endTime2 := now.AddDate(0, 0, -3).Truncate(time.Hour)
|
||||||
|
|
||||||
// kline query is exclusive
|
// kline query is exclusive
|
||||||
err = service.SyncKLineByInterval(ctx, ex, symbol, types.Interval1h, startTime1.Add(-time.Second), endTime1.Add(time.Second))
|
err = service.syncKLineByInterval(ctx, ex, symbol, types.Interval1h, startTime1.Add(-time.Second), endTime1.Add(time.Second))
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
err = service.SyncKLineByInterval(ctx, ex, symbol, types.Interval1h, startTime2.Add(-time.Second), endTime2.Add(time.Second))
|
err = service.syncKLineByInterval(ctx, ex, symbol, types.Interval1h, startTime2.Add(-time.Second), endTime2.Add(time.Second))
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
t1, t2, err := service.QueryExistingDataRange(ctx, ex, symbol, types.Interval1h)
|
t1, t2, err := service.queryExistingDataRange(ctx, ex, symbol, types.Interval1h)
|
||||||
if assert.NoError(t, err) {
|
if assert.NoError(t, err) {
|
||||||
assert.Equal(t, startTime1, t1.Time(), "start time point should match")
|
assert.Equal(t, startTime1, t1.Time(), "start time point should match")
|
||||||
assert.Equal(t, endTime2, t2.Time(), "end time point should match")
|
assert.Equal(t, endTime2, t2.Time(), "end time point should match")
|
||||||
}
|
}
|
||||||
|
|
||||||
timeRanges, err := service.FindMissingTimeRanges(ctx, ex, symbol, types.Interval1h, startTime1, endTime2)
|
timeRanges, err := service.findMissingTimeRanges(ctx, ex, symbol, types.Interval1h, startTime1, endTime2)
|
||||||
if assert.NoError(t, err) {
|
if assert.NoError(t, err) {
|
||||||
assert.NotEmpty(t, timeRanges)
|
assert.NotEmpty(t, timeRanges)
|
||||||
assert.Len(t, timeRanges, 1, "should find one missing time range")
|
assert.Len(t, timeRanges, 1, "should find one missing time range")
|
||||||
|
@ -176,11 +176,11 @@ func TestBacktestService_FindMissingTimeRanges(t *testing.T) {
|
||||||
log.SetLevel(log.DebugLevel)
|
log.SetLevel(log.DebugLevel)
|
||||||
|
|
||||||
for _, timeRange := range timeRanges {
|
for _, timeRange := range timeRanges {
|
||||||
err = service.SyncKLineByInterval(ctx, ex, symbol, types.Interval1h, timeRange.Start.Add(time.Second), timeRange.End.Add(-time.Second))
|
err = service.syncKLineByInterval(ctx, ex, symbol, types.Interval1h, timeRange.Start.Add(time.Second), timeRange.End.Add(-time.Second))
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
timeRanges, err = service.FindMissingTimeRanges(ctx, ex, symbol, types.Interval1h, startTime1, endTime2)
|
timeRanges, err = service.findMissingTimeRanges(ctx, ex, symbol, types.Interval1h, startTime1, endTime2)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.Empty(t, timeRanges, "after partial sync, missing time ranges should be back-filled")
|
assert.Empty(t, timeRanges, "after partial sync, missing time ranges should be back-filled")
|
||||||
}
|
}
|
|
@ -66,7 +66,7 @@ func RunBacktest(t *testing.T, strategy bbgo.SingleExchangeStrategy) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
backtestService := &service.BacktestService{DB: environ.DatabaseService.DB}
|
backtestService := service.NewBacktestService(environ.DatabaseService.DB)
|
||||||
defer func() {
|
defer func() {
|
||||||
err := environ.DatabaseService.DB.Close()
|
err := environ.DatabaseService.DB.Close()
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
|
@ -64,6 +64,61 @@ func (i Interval) Duration() time.Duration {
|
||||||
return time.Duration(i.Milliseconds()) * time.Millisecond
|
return time.Duration(i.Milliseconds()) * time.Millisecond
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Truncate determines the candle open time from a given timestamp
|
||||||
|
// eg interval 1 hour and tick at timestamp 00:58:45 will return timestamp shifted to 00:00:00
|
||||||
|
func (i Interval) Truncate(ts time.Time) (start time.Time) {
|
||||||
|
switch i {
|
||||||
|
case Interval1s:
|
||||||
|
return ts.Truncate(time.Second)
|
||||||
|
case Interval1m:
|
||||||
|
return ts.Truncate(time.Minute)
|
||||||
|
case Interval3m:
|
||||||
|
return shiftMinute(ts, 3)
|
||||||
|
case Interval5m:
|
||||||
|
return shiftMinute(ts, 5)
|
||||||
|
case Interval15m:
|
||||||
|
return shiftMinute(ts, 15)
|
||||||
|
case Interval30m:
|
||||||
|
return shiftMinute(ts, 30)
|
||||||
|
case Interval1h:
|
||||||
|
return ts.Truncate(time.Hour)
|
||||||
|
case Interval2h:
|
||||||
|
return shiftHour(ts, 2)
|
||||||
|
case Interval4h:
|
||||||
|
return shiftHour(ts, 4)
|
||||||
|
case Interval6h:
|
||||||
|
return shiftHour(ts, 6)
|
||||||
|
case Interval12h:
|
||||||
|
return shiftHour(ts, 12)
|
||||||
|
case Interval1d:
|
||||||
|
return ts.Truncate(time.Hour * 24)
|
||||||
|
case Interval3d:
|
||||||
|
return shiftDay(ts, 3)
|
||||||
|
case Interval1w:
|
||||||
|
return shiftDay(ts, 7)
|
||||||
|
case Interval2w:
|
||||||
|
return shiftDay(ts, 14)
|
||||||
|
case Interval1mo:
|
||||||
|
return time.Date(ts.Year(), ts.Month(), 0, 0, 0, 0, 0, time.UTC)
|
||||||
|
}
|
||||||
|
return start
|
||||||
|
}
|
||||||
|
|
||||||
|
func shiftDay(ts time.Time, shift int) time.Time {
|
||||||
|
day := ts.Day() - (ts.Day() % shift)
|
||||||
|
return time.Date(ts.Year(), ts.Month(), day, 0, 0, 0, 0, ts.Location())
|
||||||
|
}
|
||||||
|
|
||||||
|
func shiftHour(ts time.Time, shift int) time.Time {
|
||||||
|
hour := ts.Hour() - (ts.Hour() % shift)
|
||||||
|
return time.Date(ts.Year(), ts.Month(), ts.Day(), hour, 0, 0, 0, ts.Location())
|
||||||
|
}
|
||||||
|
|
||||||
|
func shiftMinute(ts time.Time, shift int) time.Time {
|
||||||
|
minute := ts.Minute() - (ts.Minute() % shift)
|
||||||
|
return time.Date(ts.Year(), ts.Month(), ts.Day(), ts.Hour(), minute, 0, 0, ts.Location())
|
||||||
|
}
|
||||||
|
|
||||||
func (i *Interval) UnmarshalJSON(b []byte) (err error) {
|
func (i *Interval) UnmarshalJSON(b []byte) (err error) {
|
||||||
var a string
|
var a string
|
||||||
err = json.Unmarshal(b, &a)
|
err = json.Unmarshal(b, &a)
|
||||||
|
|
|
@ -2,10 +2,24 @@ package types
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestTruncate(t *testing.T) {
|
||||||
|
ts := time.Date(2023, 11, 5, 17, 36, 43, 716, time.UTC)
|
||||||
|
expectedDay := time.Date(ts.Year(), ts.Month(), ts.Day(), 0, 0, 0, 0, time.UTC)
|
||||||
|
assert.Equal(t, expectedDay, Interval1d.Truncate(ts))
|
||||||
|
expected2h := time.Date(ts.Year(), ts.Month(), ts.Day(), 16, 0, 0, 0, time.UTC)
|
||||||
|
assert.Equal(t, expected2h, Interval2h.Truncate(ts))
|
||||||
|
expectedHour := time.Date(ts.Year(), ts.Month(), ts.Day(), 17, 0, 0, 0, time.UTC)
|
||||||
|
assert.Equal(t, expectedHour, Interval1h.Truncate(ts))
|
||||||
|
expected30m := time.Date(ts.Year(), ts.Month(), ts.Day(), 17, 30, 0, 0, time.UTC)
|
||||||
|
assert.Equal(t, expected30m, Interval30m.Truncate(ts))
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseInterval(t *testing.T) {
|
func TestParseInterval(t *testing.T) {
|
||||||
assert.Equal(t, ParseInterval("1s"), 1)
|
assert.Equal(t, ParseInterval("1s"), 1)
|
||||||
assert.Equal(t, ParseInterval("3m"), 3*60)
|
assert.Equal(t, ParseInterval("3m"), 3*60)
|
||||||
|
|
|
@ -37,6 +37,15 @@ func NewMillisecondTimestampFromInt(i int64) MillisecondTimestamp {
|
||||||
return MillisecondTimestamp(time.Unix(0, i*int64(time.Millisecond)))
|
return MillisecondTimestamp(time.Unix(0, i*int64(time.Millisecond)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ParseMillisecondTimestamp(a string) (ts MillisecondTimestamp, err error) {
|
||||||
|
m, err := strconv.ParseInt(a, 10, 64) // startTime
|
||||||
|
if err != nil {
|
||||||
|
return ts, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return NewMillisecondTimestampFromInt(m), nil
|
||||||
|
}
|
||||||
|
|
||||||
func MustParseMillisecondTimestamp(a string) MillisecondTimestamp {
|
func MustParseMillisecondTimestamp(a string) MillisecondTimestamp {
|
||||||
m, err := strconv.ParseInt(a, 10, 64) // startTime
|
m, err := strconv.ParseInt(a, 10, 64) // startTime
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -1,151 +0,0 @@
|
||||||
package types
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
|
|
||||||
"gonum.org/v1/gonum/stat"
|
|
||||||
|
|
||||||
"github.com/c9s/bbgo/pkg/datatype/floats"
|
|
||||||
"github.com/c9s/bbgo/pkg/fixedpoint"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// DailyToAnnualFactor is the factor to scale daily observations to annual.
|
|
||||||
// Commonly defined as the number of public market trading days in a year.
|
|
||||||
DailyToAnnualFactor = 252 // todo does this apply to crypto at all?
|
|
||||||
)
|
|
||||||
|
|
||||||
// AnnualHistoricVolatility is the historic volatility of the equity curve as annualized std dev.
|
|
||||||
func AnnualHistoricVolatility(data Series) float64 {
|
|
||||||
var sd = Stdev(data, data.Length(), 1)
|
|
||||||
return sd * math.Sqrt(DailyToAnnualFactor)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CAGR is the Compound Annual Growth Rate of the equity curve.
|
|
||||||
func CAGR(initial, final float64, days int) float64 {
|
|
||||||
var (
|
|
||||||
growthRate = (final - initial) / initial
|
|
||||||
x = 1 + growthRate
|
|
||||||
y = 365.0 / float64(days)
|
|
||||||
)
|
|
||||||
return math.Pow(x, y) - 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// measures of risk-adjusted return based on drawdown risk
|
|
||||||
|
|
||||||
// calmar ratio - discounts expected excess return of a portfolio by the
|
|
||||||
// worst expected maximum draw down for that portfolio
|
|
||||||
// CR = E(re)/MD1 = (E(r) - rf) / MD1
|
|
||||||
func CalmarRatio(cagr, maxDrawdown float64) float64 {
|
|
||||||
return cagr / maxDrawdown
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sterling ratio
|
|
||||||
// discounts the expected excess return of a portfolio by the average of the N worst
|
|
||||||
// expected maximum drawdowns for that portfolio
|
|
||||||
// CR = E(re) / (1/N)(sum MDi)
|
|
||||||
func SterlingRatio(cagr, avgDrawdown float64) float64 {
|
|
||||||
return cagr / avgDrawdown
|
|
||||||
}
|
|
||||||
|
|
||||||
// Burke Ratio
|
|
||||||
// similar to sterling, but less sensitive to outliers
|
|
||||||
// discounts the expected excess return of a portfolio by the square root of the average
|
|
||||||
// of the N worst expected maximum drawdowns for that portfolio
|
|
||||||
// BR = E(re) / ((1/N)(sum MD^2))^0.5 ---> smoothing, can take roots, logs etc
|
|
||||||
func BurkeRatio(cagr, avgDrawdownSquared float64) float64 {
|
|
||||||
return cagr / math.Sqrt(avgDrawdownSquared)
|
|
||||||
}
|
|
||||||
|
|
||||||
// KellyCriterion the famous method for trade sizing.
|
|
||||||
func KellyCriterion(profitFactor, winP fixedpoint.Value) fixedpoint.Value {
|
|
||||||
return profitFactor.Mul(winP).Sub(fixedpoint.One.Sub(winP)).Div(profitFactor)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PRR (Pessimistic Return Ratio) is the profit factor with a penalty for a lower number of roundturns.
|
|
||||||
func PRR(profit, loss, winningN, losingN fixedpoint.Value) fixedpoint.Value {
|
|
||||||
var (
|
|
||||||
winF = 1 / math.Sqrt(1+winningN.Float64())
|
|
||||||
loseF = 1 / math.Sqrt(1+losingN.Float64())
|
|
||||||
)
|
|
||||||
return fixedpoint.NewFromFloat((1 - winF) / (1 + loseF) * (1 + profit.Float64()) / (1 + loss.Float64()))
|
|
||||||
}
|
|
||||||
|
|
||||||
// StatN returns the statistically significant number of samples required based on the distribution of a series.
|
|
||||||
// From: https://www.elitetrader.com/et/threads/minimum-number-of-roundturns-required-for-backtesting-results-to-be-trusted.356588/page-2
|
|
||||||
func StatN(xs floats.Slice) (sn, se fixedpoint.Value) {
|
|
||||||
var (
|
|
||||||
sd = Stdev(xs, xs.Length(), 1)
|
|
||||||
m = Mean(xs)
|
|
||||||
statn = math.Pow(4*(sd/m), 2)
|
|
||||||
stdErr = stat.StdErr(sd, float64(xs.Length()))
|
|
||||||
)
|
|
||||||
return fixedpoint.NewFromFloat(statn), fixedpoint.NewFromFloat(stdErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// OptimalF is a function that returns the 'OptimalF' for a series of trade returns as defined by Ralph Vince.
|
|
||||||
// It is a method for sizing positions to maximize geometric return whilst accounting for biggest trading loss.
|
|
||||||
// See: https://www.investopedia.com/terms/o/optimalf.asp
|
|
||||||
// Param roundturns is the series of profits (-ve amount for losses) for each trade
|
|
||||||
func OptimalF(roundturns floats.Slice) fixedpoint.Value {
|
|
||||||
var (
|
|
||||||
maxTWR, optimalF float64
|
|
||||||
maxLoss = roundturns.Min()
|
|
||||||
)
|
|
||||||
for i := 1.0; i <= 100.0; i++ {
|
|
||||||
twr := 1.0
|
|
||||||
f := i / 100
|
|
||||||
for j := range roundturns {
|
|
||||||
if roundturns[j] == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
hpr := 1 + f*(-roundturns[j]/maxLoss)
|
|
||||||
twr *= hpr
|
|
||||||
}
|
|
||||||
if twr > maxTWR {
|
|
||||||
maxTWR = twr
|
|
||||||
optimalF = f
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return fixedpoint.NewFromFloat(optimalF)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NN (Not Number) returns y if x is NaN or Inf.
|
|
||||||
func NN(x, y float64) float64 {
|
|
||||||
if math.IsNaN(x) || math.IsInf(x, 0) {
|
|
||||||
return y
|
|
||||||
}
|
|
||||||
return x
|
|
||||||
}
|
|
||||||
|
|
||||||
// NNZ (Not Number or Zero) returns y if x is NaN or Inf or Zero.
|
|
||||||
func NNZ(x, y float64) float64 {
|
|
||||||
if NN(x, y) == y || x == 0 {
|
|
||||||
return y
|
|
||||||
}
|
|
||||||
return x
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute the drawdown function associated to a portfolio equity curve,
|
|
||||||
// also called the portfolio underwater equity curve.
|
|
||||||
// Portfolio Optimization with Drawdown Constraints, Chekhlov et al., 2000
|
|
||||||
// http://papers.ssrn.com/sol3/papers.cfm?abstract_id=223323
|
|
||||||
func Drawdown(equityCurve floats.Slice) floats.Slice {
|
|
||||||
// Initialize highWaterMark
|
|
||||||
highWaterMark := math.Inf(-1)
|
|
||||||
|
|
||||||
// Create ddVector with the same length as equityCurve
|
|
||||||
ddVector := make([]float64, len(equityCurve))
|
|
||||||
|
|
||||||
// Loop over all the values to compute the drawdown vector
|
|
||||||
for i := 0; i < len(equityCurve); i++ {
|
|
||||||
if equityCurve[i] > highWaterMark {
|
|
||||||
highWaterMark = equityCurve[i]
|
|
||||||
}
|
|
||||||
|
|
||||||
ddVector[i] = (highWaterMark - equityCurve[i]) / highWaterMark
|
|
||||||
}
|
|
||||||
|
|
||||||
return ddVector
|
|
||||||
}
|
|
|
@ -1,56 +0,0 @@
|
||||||
package types
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
|
|
||||||
"github.com/c9s/bbgo/pkg/datatype/floats"
|
|
||||||
"github.com/c9s/bbgo/pkg/fixedpoint"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCAGR(t *testing.T) {
|
|
||||||
giveInitial := 1000.0
|
|
||||||
giveFinal := 2500.0
|
|
||||||
giveDays := 190
|
|
||||||
want := 4.81
|
|
||||||
act := CAGR(giveInitial, giveFinal, giveDays)
|
|
||||||
assert.InDelta(t, want, act, 0.01)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestKellyCriterion(t *testing.T) {
|
|
||||||
var (
|
|
||||||
giveProfitFactor = fixedpoint.NewFromFloat(1.6)
|
|
||||||
giveWinP = fixedpoint.NewFromFloat(0.7)
|
|
||||||
want = 0.51
|
|
||||||
act = KellyCriterion(giveProfitFactor, giveWinP)
|
|
||||||
)
|
|
||||||
assert.InDelta(t, want, act.Float64(), 0.01)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnnualHistoricVolatility(t *testing.T) {
|
|
||||||
var (
|
|
||||||
give = floats.Slice{0.1, 0.2, -0.15, 0.1, 0.8, -0.3, 0.2}
|
|
||||||
want = 5.51
|
|
||||||
act = AnnualHistoricVolatility(give)
|
|
||||||
)
|
|
||||||
assert.InDelta(t, want, act, 0.01)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOptimalF(t *testing.T) {
|
|
||||||
roundturns := floats.Slice{10, 20, 50, -10, 40, -40}
|
|
||||||
f := OptimalF(roundturns)
|
|
||||||
assert.EqualValues(t, 0.45, f)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDrawdown(t *testing.T) {
|
|
||||||
roundturns := floats.Slice{100, 50, 100}
|
|
||||||
expected := []float64{.0, .5, .0}
|
|
||||||
drawdown := Drawdown(roundturns)
|
|
||||||
assert.EqualValues(t, 0.5, drawdown.Max())
|
|
||||||
assert.EqualValues(t, 0.16666666666666666, drawdown.Average())
|
|
||||||
assert.EqualValues(t, 0.08333333333333333, drawdown.AverageSquared())
|
|
||||||
for i, v := range expected {
|
|
||||||
assert.EqualValues(t, v, drawdown[i])
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -14,10 +14,38 @@ import (
|
||||||
"github.com/c9s/bbgo/pkg/fixedpoint"
|
"github.com/c9s/bbgo/pkg/fixedpoint"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
type IntervalProfitCollector struct {
|
||||||
ErrStartTimeNotValid = "No valid start time. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?"
|
Interval Interval `json:"interval"`
|
||||||
ErrProfitArrEmpty = "profits array empty. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?"
|
Profits *floats.Slice `json:"profits"`
|
||||||
)
|
Timestamp *floats.Slice `json:"timestamp"`
|
||||||
|
tmpTime time.Time `json:"tmpTime"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewIntervalProfitCollector(i Interval, startTime time.Time) *IntervalProfitCollector {
|
||||||
|
return &IntervalProfitCollector{Interval: i, tmpTime: startTime, Profits: &floats.Slice{1.}, Timestamp: &floats.Slice{float64(startTime.Unix())}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update the collector by every traded profit
|
||||||
|
func (s *IntervalProfitCollector) Update(profit *Profit) {
|
||||||
|
if s.tmpTime.IsZero() {
|
||||||
|
panic("No valid start time. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?")
|
||||||
|
} else {
|
||||||
|
duration := s.Interval.Duration()
|
||||||
|
if profit.TradedAt.Before(s.tmpTime.Add(duration)) {
|
||||||
|
(*s.Profits)[len(*s.Profits)-1] *= 1. + profit.NetProfitMargin.Float64()
|
||||||
|
} else {
|
||||||
|
for {
|
||||||
|
s.Profits.Update(1.)
|
||||||
|
s.tmpTime = s.tmpTime.Add(duration)
|
||||||
|
s.Timestamp.Update(float64(s.tmpTime.Unix()))
|
||||||
|
if profit.TradedAt.Before(s.tmpTime.Add(duration)) {
|
||||||
|
(*s.Profits)[len(*s.Profits)-1] *= 1. + profit.NetProfitMargin.Float64()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type ProfitReport struct {
|
type ProfitReport struct {
|
||||||
StartTime time.Time `json:"startTime"`
|
StartTime time.Time `json:"startTime"`
|
||||||
|
@ -33,55 +61,6 @@ func (s ProfitReport) String() string {
|
||||||
return string(b)
|
return string(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
type IntervalProfitCollector struct {
|
|
||||||
Interval Interval `json:"interval"`
|
|
||||||
Profits floats.Slice `json:"profits"`
|
|
||||||
TimeInMarket []time.Duration `json:"timeInMarket"`
|
|
||||||
Timestamp floats.Slice `json:"timestamp"`
|
|
||||||
tmpTime time.Time `json:"tmpTime"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewIntervalProfitCollector(i Interval, startTime time.Time) *IntervalProfitCollector {
|
|
||||||
return &IntervalProfitCollector{Interval: i, tmpTime: startTime, Profits: floats.Slice{1.}, Timestamp: floats.Slice{float64(startTime.Unix())}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update the collector by every traded profit
|
|
||||||
func (s *IntervalProfitCollector) Update(profit *Profit) {
|
|
||||||
if s.tmpTime.IsZero() {
|
|
||||||
panic(ErrStartTimeNotValid)
|
|
||||||
} else {
|
|
||||||
s.TimeInMarket = append(s.TimeInMarket, profit.TradedAt.Sub(profit.PositionOpenedAt))
|
|
||||||
duration := s.Interval.Duration()
|
|
||||||
if profit.TradedAt.Before(s.tmpTime.Add(duration)) {
|
|
||||||
(s.Profits)[len(s.Profits)-1] *= 1. + profit.NetProfitMargin.Float64()
|
|
||||||
} else {
|
|
||||||
for {
|
|
||||||
s.Profits.Update(1.)
|
|
||||||
s.tmpTime = s.tmpTime.Add(duration)
|
|
||||||
s.Timestamp.Update(float64(s.tmpTime.Unix()))
|
|
||||||
if profit.TradedAt.Before(s.tmpTime.Add(duration)) {
|
|
||||||
(s.Profits)[len(s.Profits)-1] *= 1. + profit.NetProfitMargin.Float64()
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine average and total time spend in market
|
|
||||||
func (s *IntervalProfitCollector) GetTimeInMarket() (avgHoldSec, totalTimeInMarketSec int64) {
|
|
||||||
if s.Profits == nil {
|
|
||||||
return 0, 0
|
|
||||||
}
|
|
||||||
l := len(s.TimeInMarket)
|
|
||||||
for i := 0; i < l; i++ {
|
|
||||||
d := s.TimeInMarket[i]
|
|
||||||
totalTimeInMarketSec += int64(d / time.Millisecond)
|
|
||||||
}
|
|
||||||
avgHoldSec = totalTimeInMarketSec / int64(l)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get all none-profitable intervals
|
// Get all none-profitable intervals
|
||||||
func (s *IntervalProfitCollector) GetNonProfitableIntervals() (result []ProfitReport) {
|
func (s *IntervalProfitCollector) GetNonProfitableIntervals() (result []ProfitReport) {
|
||||||
if s.Profits == nil {
|
if s.Profits == nil {
|
||||||
|
@ -113,9 +92,9 @@ func (s *IntervalProfitCollector) GetProfitableIntervals() (result []ProfitRepor
|
||||||
// Get number of profitable traded intervals
|
// Get number of profitable traded intervals
|
||||||
func (s *IntervalProfitCollector) GetNumOfProfitableIntervals() (profit int) {
|
func (s *IntervalProfitCollector) GetNumOfProfitableIntervals() (profit int) {
|
||||||
if s.Profits == nil {
|
if s.Profits == nil {
|
||||||
panic(ErrProfitArrEmpty)
|
panic("profits array empty. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?")
|
||||||
}
|
}
|
||||||
for _, v := range s.Profits {
|
for _, v := range *s.Profits {
|
||||||
if v > 1. {
|
if v > 1. {
|
||||||
profit += 1
|
profit += 1
|
||||||
}
|
}
|
||||||
|
@ -127,9 +106,9 @@ func (s *IntervalProfitCollector) GetNumOfProfitableIntervals() (profit int) {
|
||||||
// (no trade within the interval or pnl = 0 will be also included here)
|
// (no trade within the interval or pnl = 0 will be also included here)
|
||||||
func (s *IntervalProfitCollector) GetNumOfNonProfitableIntervals() (nonprofit int) {
|
func (s *IntervalProfitCollector) GetNumOfNonProfitableIntervals() (nonprofit int) {
|
||||||
if s.Profits == nil {
|
if s.Profits == nil {
|
||||||
panic(ErrProfitArrEmpty)
|
panic("profits array empty. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?")
|
||||||
}
|
}
|
||||||
for _, v := range s.Profits {
|
for _, v := range *s.Profits {
|
||||||
if v <= 1. {
|
if v <= 1. {
|
||||||
nonprofit += 1
|
nonprofit += 1
|
||||||
}
|
}
|
||||||
|
@ -141,11 +120,10 @@ func (s *IntervalProfitCollector) GetNumOfNonProfitableIntervals() (nonprofit in
|
||||||
// no smart sharpe ON for the calculated result
|
// no smart sharpe ON for the calculated result
|
||||||
func (s *IntervalProfitCollector) GetSharpe() float64 {
|
func (s *IntervalProfitCollector) GetSharpe() float64 {
|
||||||
if s.tmpTime.IsZero() {
|
if s.tmpTime.IsZero() {
|
||||||
panic(ErrStartTimeNotValid)
|
panic("No valid start time. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?")
|
||||||
}
|
}
|
||||||
if s.Profits == nil {
|
if s.Profits == nil {
|
||||||
panic(ErrStartTimeNotValid)
|
panic("profits array empty. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?")
|
||||||
|
|
||||||
}
|
}
|
||||||
return Sharpe(Sub(s.Profits, 1.), s.Profits.Length(), true, false)
|
return Sharpe(Sub(s.Profits, 1.), s.Profits.Length(), true, false)
|
||||||
}
|
}
|
||||||
|
@ -154,10 +132,10 @@ func (s *IntervalProfitCollector) GetSharpe() float64 {
|
||||||
// No risk-free return rate and smart sortino OFF for the calculated result.
|
// No risk-free return rate and smart sortino OFF for the calculated result.
|
||||||
func (s *IntervalProfitCollector) GetSortino() float64 {
|
func (s *IntervalProfitCollector) GetSortino() float64 {
|
||||||
if s.tmpTime.IsZero() {
|
if s.tmpTime.IsZero() {
|
||||||
panic(ErrStartTimeNotValid)
|
panic("No valid start time. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?")
|
||||||
}
|
}
|
||||||
if s.Profits == nil {
|
if s.Profits == nil {
|
||||||
panic(ErrProfitArrEmpty)
|
panic("profits array empty. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?")
|
||||||
}
|
}
|
||||||
return Sortino(Sub(s.Profits, 1.), 0., s.Profits.Length(), true, false)
|
return Sortino(Sub(s.Profits, 1.), 0., s.Profits.Length(), true, false)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user