package cmd import ( "bufio" "context" "fmt" "os" "path/filepath" "strings" "syscall" "time" "github.com/fatih/color" "github.com/google/uuid" "github.com/c9s/bbgo/pkg/cmd/cmdutil" "github.com/c9s/bbgo/pkg/core" "github.com/c9s/bbgo/pkg/data/tsv" "github.com/c9s/bbgo/pkg/util" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/c9s/bbgo/pkg/accounting/pnl" "github.com/c9s/bbgo/pkg/backtest" "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/exchange" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/service" "github.com/c9s/bbgo/pkg/types" ) func init() { BacktestCmd.Flags().Bool("sync", false, "sync backtest data") BacktestCmd.Flags().Bool("sync-only", false, "sync backtest data only, do not run backtest") BacktestCmd.Flags().String("sync-from", "", "sync backtest data from the given time, which will override the time range in the backtest config") BacktestCmd.Flags().String("sync-exchange", "", "specify only one exchange to sync backtest data") BacktestCmd.Flags().String("session", "", "specify only one exchange session to run backtest") BacktestCmd.Flags().Bool("verify", false, "verify the kline back-test data") BacktestCmd.Flags().Bool("base-asset-baseline", false, "use base asset performance as the competitive baseline performance") BacktestCmd.Flags().CountP("verbose", "v", "verbose level") BacktestCmd.Flags().Bool("force", false, "force execution without confirm") BacktestCmd.Flags().String("output", "", "the report output directory") BacktestCmd.Flags().Bool("subdir", false, "generate report in the sub-directory of the output directory") RootCmd.AddCommand(BacktestCmd) } var BacktestCmd = &cobra.Command{ Use: "backtest", Short: "run backtest with strategies", SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { verboseCnt, err := cmd.Flags().GetCount("verbose") if err != nil { return err } if viper.GetBool("debug") { verboseCnt = 2 } configFile, err := cmd.Flags().GetString("config") if err != nil { return err } if len(configFile) == 0 { return errors.New("--config option is required") } wantBaseAssetBaseline, err := cmd.Flags().GetBool("base-asset-baseline") if err != nil { return err } wantSync, err := cmd.Flags().GetBool("sync") if err != nil { return err } syncExchangeName, err := cmd.Flags().GetString("sync-exchange") if err != nil { return err } sessionName, err := cmd.Flags().GetString("session") if err != nil { return err } force, err := cmd.Flags().GetBool("force") if err != nil { return err } outputDirectory, err := cmd.Flags().GetString("output") if err != nil { return err } generatingReport := len(outputDirectory) > 0 reportFileInSubDir, err := cmd.Flags().GetBool("subdir") if err != nil { return err } syncOnly, err := cmd.Flags().GetBool("sync-only") if err != nil { return err } syncFromDateStr, err := cmd.Flags().GetString("sync-from") if err != nil { return err } shouldVerify, err := cmd.Flags().GetBool("verify") if err != nil { return err } userConfig, err := bbgo.Load(configFile, true) if err != nil { return err } if userConfig.Backtest == nil { return errors.New("backtest config is not defined") } ctx, cancel := context.WithCancel(context.Background()) defer cancel() var now = time.Now().Local() var startTime, endTime time.Time startTime = userConfig.Backtest.StartTime.Time().Local() // set default start time to the past 6 months // userConfig.Backtest.StartTime = now.AddDate(0, -6, 0).Format("2006-01-02") if userConfig.Backtest.EndTime != nil { endTime = userConfig.Backtest.EndTime.Time().Local() } else { endTime = now } // ensure that we're using local time startTime = startTime.Local() endTime = endTime.Local() log.Infof("starting backtest with startTime %s", startTime.Format(time.RFC3339)) environ := bbgo.NewEnvironment() if err := bbgo.BootstrapBacktestEnvironment(ctx, environ); err != nil { return err } if environ.DatabaseService == nil { return errors.New("database service is not enabled, please check your environment variables DB_DRIVER and DB_DSN") } backtestService := &service.BacktestService{DB: environ.DatabaseService.DB} environ.BacktestService = backtestService bbgo.SetBackTesting(backtestService) if len(sessionName) > 0 { userConfig.Backtest.Sessions = []string{sessionName} } else if len(syncExchangeName) > 0 { userConfig.Backtest.Sessions = []string{syncExchangeName} } else if len(userConfig.Backtest.Sessions) == 0 { log.Infof("backtest.sessions is not defined, using all supported exchanges: %v", types.SupportedExchanges) for _, exName := range types.SupportedExchanges { userConfig.Backtest.Sessions = append(userConfig.Backtest.Sessions, exName.String()) } } var sourceExchanges = make(map[types.ExchangeName]types.Exchange) for _, name := range userConfig.Backtest.Sessions { exName, err := types.ValidExchangeName(name) if err != nil { return err } publicExchange, err := exchange.NewPublic(exName) if err != nil { return err } sourceExchanges[exName] = publicExchange // Set exchange to use futures if userConfig.Sessions[exName.String()].Futures { futuresExchange, ok := publicExchange.(types.FuturesExchange) if !ok { return fmt.Errorf("exchange %s does not support futures", publicExchange.Name()) } futuresExchange.UseFutures() } } var syncFromTime time.Time // user can override the sync from time if the option is given if len(syncFromDateStr) > 0 { syncFromTime, err = time.Parse(types.DateFormat, syncFromDateStr) if err != nil { return err } if syncFromTime.After(startTime) { return fmt.Errorf("sync-from time %s can not be latter than the backtest start time %s", syncFromTime, startTime) } syncFromTime = syncFromTime.Local() } else { // we need at least 1 month backward data for EMA and last prices syncFromTime = startTime.AddDate(0, -1, 0) log.Infof("adjusted sync start time %s to %s for backward market data", startTime, syncFromTime) } if wantSync { log.Infof("starting synchronization: %v", userConfig.Backtest.Symbols) if err := sync(ctx, userConfig, backtestService, sourceExchanges, syncFromTime, endTime); err != nil { return err } log.Info("synchronization done") if shouldVerify { err := verify(userConfig, backtestService, sourceExchanges, syncFromTime, endTime) if err != nil { return err } } if syncOnly { return nil } } if userConfig.Backtest.RecordTrades { log.Warn("!!! Trade recording is enabled for back-testing !!!") log.Warn("!!! To run back-testing, you should use an isolated database for storing back-testing trades !!!") log.Warn("!!! The trade record in the current database WILL ALL BE DELETED BEFORE THIS BACK-TESTING !!!") if !force { if !confirmation("Are you sure to continue?") { return nil } } if err := environ.TradeService.DeleteAll(); err != nil { return err } } if verboseCnt == 2 { log.SetLevel(log.DebugLevel) } else if verboseCnt > 0 { log.SetLevel(log.InfoLevel) } else { // default mode, disable strategy logging and order executor logging log.SetLevel(log.ErrorLevel) } environ.SetStartTime(startTime) // exchangeNameStr is the session name. for name, sourceExchange := range sourceExchanges { backtestExchange, err := backtest.NewExchange(sourceExchange.Name(), sourceExchange, backtestService, userConfig.Backtest) if err != nil { return errors.Wrap(err, "failed to create backtest exchange") } session := environ.AddExchange(name.String(), backtestExchange) exchangeFromConfig := userConfig.Sessions[name.String()] if exchangeFromConfig != nil { session.UseHeikinAshi = exchangeFromConfig.UseHeikinAshi session.Futures = exchangeFromConfig.Futures } } if err := environ.Init(ctx); err != nil { return err } for _, session := range environ.Sessions() { userDataStream := session.UserDataStream.(types.StandardStreamEmitter) backtestEx := session.Exchange.(*backtest.Exchange) backtestEx.MarketDataStream = session.MarketDataStream.(types.StandardStreamEmitter) backtestEx.BindUserData(userDataStream) } trader := bbgo.NewTrader(environ) if verboseCnt == 0 { trader.DisableLogging() } if err := trader.Configure(userConfig); err != nil { return err } if err := trader.Initialize(ctx); err != nil { return err } if err := trader.Run(ctx); err != nil { return err } allKLineIntervals, requiredInterval, backTestIntervals := backtest.CollectSubscriptionIntervals(environ) exchangeSources, err := backtest.InitializeExchangeSources(environ.Sessions(), startTime, endTime, requiredInterval, backTestIntervals...) if err != nil { return err } var kLineHandlers []func(k types.KLine, exSource *backtest.ExchangeDataSource) var manifests backtest.Manifests var runID = userConfig.GetSignature() + "_" + uuid.NewString() var reportDir = outputDirectory var sessionTradeStats = make(map[string]map[string]*types.TradeStats) // for each exchange session, iterate the positions and // allocate trade collector to calculate the tradeStats var tradeCollectorList []*core.TradeCollector for _, exSource := range exchangeSources { sessionName := exSource.Session.Name tradeStatsMap := make(map[string]*types.TradeStats) for usedSymbol := range exSource.Session.Positions() { market, _ := exSource.Session.Market(usedSymbol) position := types.NewPositionFromMarket(market) orderStore := core.NewOrderStore(usedSymbol) orderStore.AddOrderUpdate = true tradeCollector := core.NewTradeCollector(usedSymbol, position, orderStore) tradeStats := types.NewTradeStats(usedSymbol) tradeStats.SetIntervalProfitCollector(types.NewIntervalProfitCollector(types.Interval1d, startTime)) tradeCollector.OnProfit(func(trade types.Trade, profit *types.Profit) { if profit == nil { return } tradeStats.Add(profit) }) tradeStatsMap[usedSymbol] = tradeStats orderStore.BindStream(exSource.Session.UserDataStream) tradeCollector.BindStream(exSource.Session.UserDataStream) tradeCollectorList = append(tradeCollectorList, tradeCollector) } sessionTradeStats[sessionName] = tradeStatsMap } kLineHandlers = append(kLineHandlers, func(k types.KLine, _ *backtest.ExchangeDataSource) { if k.Interval == types.Interval1d && k.Closed { for _, collector := range tradeCollectorList { collector.Process() } } }) if generatingReport { if reportFileInSubDir { // reportDir = filepath.Join(reportDir, backtestSessionName) reportDir = filepath.Join(reportDir, runID) } if err := util.SafeMkdirAll(reportDir); err != nil { return err } startTimeStr := startTime.Format("20060102") endTimeStr := endTime.Format("20060102") kLineSubDir := strings.Join([]string{"klines", "_", startTimeStr, "-", endTimeStr}, "") kLineDataDir := filepath.Join(outputDirectory, "shared", kLineSubDir) if err := util.SafeMkdirAll(kLineDataDir); err != nil { return err } stateRecorder := backtest.NewStateRecorder(reportDir) err = trader.IterateStrategies(func(st bbgo.StrategyID) error { return stateRecorder.Scan(st.(backtest.Instance)) }) if err != nil { return err } manifests = stateRecorder.Manifests() manifests, err = rewriteManifestPaths(manifests, reportDir) if err != nil { return err } // state snapshot kLineHandlers = append(kLineHandlers, func(k types.KLine, _ *backtest.ExchangeDataSource) { // snapshot per 1m if k.Interval == types.Interval1m && k.Closed { if _, err := stateRecorder.Snapshot(); err != nil { log.WithError(err).Errorf("state record failed to snapshot the strategy state") } } }) dumper := backtest.NewKLineDumper(kLineDataDir) defer func() { if err := dumper.Close(); err != nil { log.WithError(err).Errorf("kline dumper can not close files") } }() kLineHandlers = append(kLineHandlers, func(k types.KLine, _ *backtest.ExchangeDataSource) { if err := dumper.Record(k); err != nil { log.WithError(err).Errorf("can not write kline to file") } }) // equity curve recording -- record per 1h kline equityCurveTsv, err := tsv.NewWriterFile(filepath.Join(reportDir, "equity_curve.tsv")) if err != nil { return err } defer func() { _ = equityCurveTsv.Close() }() _ = equityCurveTsv.Write([]string{ "time", "in_usd", }) defer equityCurveTsv.Flush() kLineHandlers = append(kLineHandlers, func(k types.KLine, exSource *backtest.ExchangeDataSource) { if k.Interval != types.Interval1h { return } balances, err := exSource.Exchange.QueryAccountBalances(ctx) if err != nil { log.WithError(err).Errorf("query back-test account balance error") } else { assets := balances.Assets(exSource.Session.AllLastPrices(), k.EndTime.Time()) _ = equityCurveTsv.Write([]string{ k.EndTime.Time().Format(time.RFC1123), assets.InUSD().String(), }) } }) ordersTsv, err := tsv.NewWriterFile(filepath.Join(reportDir, "orders.tsv")) if err != nil { return err } defer func() { _ = ordersTsv.Close() }() _ = ordersTsv.Write(types.Order{}.CsvHeader()) for _, exSource := range exchangeSources { exSource.Session.UserDataStream.OnOrderUpdate(func(order types.Order) { if order.Status == types.OrderStatusFilled { for _, record := range order.CsvRecords() { _ = ordersTsv.Write(record) } } }) } } runCtx, cancelRun := context.WithCancel(ctx) for _, exK := range exchangeSources { exK.Callbacks = kLineHandlers } go func() { defer cancelRun() // Optimize back-test speed for single exchange source var numOfExchangeSources = len(exchangeSources) if numOfExchangeSources == 1 { exSource := exchangeSources[0] for k := range exSource.C { exSource.Exchange.ConsumeKLine(k, requiredInterval) } if err := exSource.Exchange.CloseMarketData(); err != nil { log.WithError(err).Errorf("close market data error") } return } RunMultiExchangeData: for { for _, exK := range exchangeSources { k, more := <-exK.C if !more { if err := exK.Exchange.CloseMarketData(); err != nil { log.WithError(err).Errorf("close market data error") return } break RunMultiExchangeData } exK.Exchange.ConsumeKLine(k, requiredInterval) } } }() cmdutil.WaitForSignal(runCtx, syscall.SIGINT, syscall.SIGTERM) log.Infof("shutting down trader...") gracefulShutdownPeriod := 30 * time.Second shtCtx, cancelShutdown := context.WithTimeout(bbgo.NewTodoContextWithExistingIsolation(ctx), gracefulShutdownPeriod) bbgo.Shutdown(shtCtx) cancelShutdown() // put the logger back to print the pnl log.SetLevel(log.InfoLevel) // aggregate total balances initTotalBalances := types.BalanceMap{} finalTotalBalances := types.BalanceMap{} var sessionNames []string for _, session := range environ.Sessions() { sessionNames = append(sessionNames, session.Name) accountConfig := userConfig.Backtest.GetAccount(session.Name) initBalances := accountConfig.Balances.BalanceMap() initTotalBalances = initTotalBalances.Add(initBalances) finalBalances := session.GetAccount().Balances() finalTotalBalances = finalTotalBalances.Add(finalBalances) } summaryReport := &backtest.SummaryReport{ StartTime: startTime, EndTime: endTime, Sessions: sessionNames, InitialTotalBalances: initTotalBalances, FinalTotalBalances: finalTotalBalances, Manifests: manifests, Symbols: nil, } for interval := range allKLineIntervals { summaryReport.Intervals = append(summaryReport.Intervals, interval) } for _, session := range environ.Sessions() { for symbol, trades := range session.Trades { if len(trades.Trades) == 0 { log.Warnf("session has no %s trades", symbol) continue } tradeState := sessionTradeStats[session.Name][symbol] profitFactor := tradeState.ProfitFactor winningRatio := tradeState.WinningRatio intervalProfits := tradeState.IntervalProfits[types.Interval1d] symbolReport, err := createSymbolReport(userConfig, session, symbol, trades.Copy(), intervalProfits, profitFactor, winningRatio) if err != nil { return err } summaryReport.Symbols = append(summaryReport.Symbols, symbol) summaryReport.SymbolReports = append(summaryReport.SymbolReports, *symbolReport) summaryReport.TotalProfit = symbolReport.PnL.Profit summaryReport.TotalUnrealizedProfit = symbolReport.PnL.UnrealizedProfit summaryReport.InitialEquityValue = summaryReport.InitialEquityValue.Add(symbolReport.InitialEquityValue()) summaryReport.FinalEquityValue = summaryReport.FinalEquityValue.Add(symbolReport.FinalEquityValue()) summaryReport.TotalGrossProfit.Add(symbolReport.PnL.GrossProfit) summaryReport.TotalGrossLoss.Add(symbolReport.PnL.GrossLoss) // write report to a file if generatingReport { reportFileName := fmt.Sprintf("symbol_report_%s_%s.json", session.Name, symbol) if err := util.WriteJsonFile(filepath.Join(reportDir, reportFileName), &symbolReport); err != nil { return err } } } } if generatingReport { summaryReportFile := filepath.Join(reportDir, "summary.json") // output summary report filepath to stdout, so that our optimizer can read from it fmt.Println(summaryReportFile) if err := util.WriteJsonFile(summaryReportFile, summaryReport); err != nil { return errors.Wrapf(err, "can not write summary report json file: %s", summaryReportFile) } configJsonFile := filepath.Join(reportDir, "config.json") if err := util.WriteJsonFile(configJsonFile, userConfig); err != nil { return errors.Wrapf(err, "can not write config json file: %s", configJsonFile) } // append report index if reportFileInSubDir { if err := backtest.AddReportIndexRun(outputDirectory, backtest.Run{ ID: runID, Config: userConfig, Time: time.Now(), }); err != nil { return err } } } else { color.Green("BACK-TEST REPORT") color.Green("===============================================\n") color.Green("START TIME: %s\n", startTime.Format(time.RFC1123)) color.Green("END TIME: %s\n", endTime.Format(time.RFC1123)) color.Green("INITIAL TOTAL BALANCE: %v\n", initTotalBalances) color.Green("FINAL TOTAL BALANCE: %v\n", finalTotalBalances) for _, symbolReport := range summaryReport.SymbolReports { symbolReport.Print(wantBaseAssetBaseline) } } return nil }, } func createSymbolReport( userConfig *bbgo.Config, session *bbgo.ExchangeSession, symbol string, trades []types.Trade, intervalProfit *types.IntervalProfitCollector, profitFactor, winningRatio fixedpoint.Value, ) ( *backtest.SessionSymbolReport, error, ) { backtestExchange, ok := session.Exchange.(*backtest.Exchange) if !ok { return nil, fmt.Errorf("unexpected error, exchange instance is not a backtest exchange") } market, ok := session.Market(symbol) if !ok { return nil, fmt.Errorf("market not found: %s, %s", symbol, session.Exchange.Name()) } startPrice, ok := session.StartPrice(symbol) if !ok { return nil, fmt.Errorf("start price not found: %s, %s. run --sync first", symbol, session.Exchange.Name()) } lastPrice, ok := session.LastPrice(symbol) if !ok { return nil, fmt.Errorf("last price not found: %s, %s", symbol, session.Exchange.Name()) } calculator := &pnl.AverageCostCalculator{ TradingFeeCurrency: backtestExchange.PlatformFeeCurrency(), Market: market, } sharpeRatio := fixedpoint.NewFromFloat(intervalProfit.GetSharpe()) sortinoRatio := fixedpoint.NewFromFloat(intervalProfit.GetSortino()) report := calculator.Calculate(symbol, trades, lastPrice) accountConfig := userConfig.Backtest.GetAccount(session.Exchange.Name().String()) initBalances := accountConfig.Balances.BalanceMap() finalBalances := session.GetAccount().Balances() symbolReport := backtest.SessionSymbolReport{ Exchange: session.Exchange.Name(), Symbol: symbol, Market: market, LastPrice: lastPrice, StartPrice: startPrice, PnL: report, InitialBalances: initBalances, FinalBalances: finalBalances, // Manifests: manifests, Sharpe: sharpeRatio, Sortino: sortinoRatio, ProfitFactor: profitFactor, WinningRatio: winningRatio, } for _, s := range session.Subscriptions { symbolReport.Subscriptions = append(symbolReport.Subscriptions, s) } sessionKLineIntervals := map[types.Interval]struct{}{} for _, sub := range session.Subscriptions { if sub.Channel == types.KLineChannel { sessionKLineIntervals[sub.Options.Interval] = struct{}{} } } for interval := range sessionKLineIntervals { symbolReport.Intervals = append(symbolReport.Intervals, interval) } return &symbolReport, nil } func verify( userConfig *bbgo.Config, backtestService *service.BacktestService, sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time, ) error { for _, sourceExchange := range sourceExchanges { err := backtestService.Verify(sourceExchange, userConfig.Backtest.Symbols, startTime, endTime) if err != nil { return err } } return nil } func confirmation(s string) bool { reader := bufio.NewReader(os.Stdin) for { fmt.Printf("%s [y/N]: ", s) response, err := reader.ReadString('\n') if err != nil { log.Fatal(err) } response = strings.ToLower(strings.TrimSpace(response)) if response == "y" || response == "yes" { return true } else if response == "n" || response == "no" { return false } else { return false } } } func getExchangeIntervals(ex types.Exchange) types.IntervalMap { exCustom, ok := ex.(types.CustomIntervalProvider) if ok { return exCustom.SupportedInterval() } return types.SupportedIntervals } func sync( ctx context.Context, userConfig *bbgo.Config, backtestService *service.BacktestService, sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time, ) error { for _, symbol := range userConfig.Backtest.Symbols { for _, sourceExchange := range sourceExchanges { var supportIntervals = getExchangeIntervals(sourceExchange) if !userConfig.Backtest.SyncSecKLines { delete(supportIntervals, types.Interval1s) } // sort intervals var intervals = supportIntervals.Slice() intervals.Sort() for _, interval := range intervals { if err := backtestService.Sync(ctx, sourceExchange, symbol, interval, syncFrom, syncTo); err != nil { return err } } } } return nil } func rewriteManifestPaths(manifests backtest.Manifests, basePath string) (backtest.Manifests, error) { var filterManifests = backtest.Manifests{} for k, m := range manifests { p, err := filepath.Rel(basePath, m) if err != nil { return nil, err } filterManifests[k] = p } return filterManifests, nil }