From f19bbbc5aac5673f3ca129868b69ca700ef4dc6e Mon Sep 17 00:00:00 2001 From: Sven Woldt Date: Thu, 2 Nov 2023 02:16:23 +0100 Subject: [PATCH] add more trade stats --- pkg/backtest/report.go | 54 +++++++++++++----- pkg/cmd/backtest.go | 108 +++++++++++++++++++++++++---------- pkg/types/trade_stat.go | 107 ++++++++++++++++++++++++++++++++++ pkg/types/trade_stat_test.go | 44 ++++++++++++++ pkg/types/trade_stats.go | 64 +++++++++++++++++---- 5 files changed, 321 insertions(+), 56 deletions(-) create mode 100644 pkg/types/trade_stat.go create mode 100644 pkg/types/trade_stat_test.go diff --git a/pkg/backtest/report.go b/pkg/backtest/report.go index bce827699..ceea79455 100644 --- a/pkg/backtest/report.go +++ b/pkg/backtest/report.go @@ -68,21 +68,45 @@ func ReadSummaryReport(filename string) (*SummaryReport, error) { // SessionSymbolReport is the report per exchange session // trades are merged, collected and re-calculated type SessionSymbolReport struct { - Exchange types.ExchangeName `json:"exchange"` - Symbol string `json:"symbol,omitempty"` - Intervals []types.Interval `json:"intervals,omitempty"` - Subscriptions []types.Subscription `json:"subscriptions"` - Market types.Market `json:"market"` - LastPrice fixedpoint.Value `json:"lastPrice,omitempty"` - StartPrice fixedpoint.Value `json:"startPrice,omitempty"` - PnL *pnl.AverageCostPnLReport `json:"pnl,omitempty"` - InitialBalances types.BalanceMap `json:"initialBalances,omitempty"` - FinalBalances types.BalanceMap `json:"finalBalances,omitempty"` - Manifests Manifests `json:"manifests,omitempty"` - Sharpe fixedpoint.Value `json:"sharpeRatio"` - Sortino fixedpoint.Value `json:"sortinoRatio"` - ProfitFactor fixedpoint.Value `json:"profitFactor"` - WinningRatio fixedpoint.Value `json:"winningRatio"` + Exchange types.ExchangeName `json:"exchange"` + Symbol string `json:"symbol,omitempty"` + Intervals []types.Interval `json:"intervals,omitempty"` + Subscriptions []types.Subscription `json:"subscriptions"` + Market types.Market `json:"market"` + LastPrice fixedpoint.Value `json:"lastPrice,omitempty"` + StartPrice fixedpoint.Value `json:"startPrice,omitempty"` + PnL *pnl.AverageCostPnLReport `json:"pnl,omitempty"` + InitialBalances types.BalanceMap `json:"initialBalances,omitempty"` + FinalBalances types.BalanceMap `json:"finalBalances,omitempty"` + Manifests Manifests `json:"manifests,omitempty"` + TradeCount fixedpoint.Value `json:"tradeCount,omitempty"` + RoundTurnCount fixedpoint.Value `json:"roundTurnCount,omitempty"` + TotalNetProfit fixedpoint.Value `json:"totalNetProfit,omitempty"` + AvgNetProfit fixedpoint.Value `json:"avgNetProfit,omitempty"` + GrossProfit fixedpoint.Value `json:"grossProfit,omitempty"` + GrossLoss fixedpoint.Value `json:"grossLoss,omitempty"` + PRR fixedpoint.Value `json:"prr,omitempty"` + PercentProfitable fixedpoint.Value `json:"percentProfitable,omitempty"` + MaxProfit fixedpoint.Value `json:"maxProfit,omitempty"` + MaxLoss fixedpoint.Value `json:"maxLoss,omitempty"` + AvgProfit fixedpoint.Value `json:"avgProfit,omitempty"` + AvgLoss fixedpoint.Value `json:"avgLoss,omitempty"` + TotalTimeInMarketSec int64 `json:"totalTimeInMarketSec,omitempty"` + AvgHoldSec int64 `json:"avgHoldSec,omitempty"` + WinningCount int `json:"winningCount,omitempty"` + LosingCount int `json:"losingCount,omitempty"` + MaxLossStreak int `json:"maxLossStreak,omitempty"` + Sharpe fixedpoint.Value `json:"sharpeRatio"` + AnnualHistoricVolatility fixedpoint.Value `json:"annualHistoricVolatility,omitempty"` + CAGR fixedpoint.Value `json:"cagr,omitempty"` + Calmar fixedpoint.Value `json:"calmar,omitempty"` + Kelly fixedpoint.Value `json:"kelly,omitempty"` + OptimalF fixedpoint.Value `json:"optimalF,omitempty"` + StatN fixedpoint.Value `json:"statN,omitempty"` + StatNStdErr fixedpoint.Value `json:"statNStdErr,omitempty"` + Sortino fixedpoint.Value `json:"sortinoRatio"` + ProfitFactor fixedpoint.Value `json:"profitFactor"` + WinningRatio fixedpoint.Value `json:"winningRatio"` } func (r *SessionSymbolReport) InitialEquityValue() fixedpoint.Value { diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 7ed612ee3..f02b1aae0 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "fmt" + "math" "os" "path/filepath" "strings" @@ -12,12 +13,6 @@ import ( "github.com/fatih/color" "github.com/google/uuid" - - "github.com/c9s/bbgo/pkg/cmd/cmdutil" - "github.com/c9s/bbgo/pkg/core" - "github.com/c9s/bbgo/pkg/data/tsv" - "github.com/c9s/bbgo/pkg/util" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -26,10 +21,14 @@ import ( "github.com/c9s/bbgo/pkg/accounting/pnl" "github.com/c9s/bbgo/pkg/backtest" "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/cmd/cmdutil" + "github.com/c9s/bbgo/pkg/core" + "github.com/c9s/bbgo/pkg/data/tsv" "github.com/c9s/bbgo/pkg/exchange" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/service" "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/util" ) func init() { @@ -528,12 +527,9 @@ var BacktestCmd = &cobra.Command{ for _, session := range environ.Sessions() { for symbol, trades := range session.Trades { - tradeState := sessionTradeStats[session.Name][symbol] - profitFactor := tradeState.ProfitFactor - winningRatio := tradeState.WinningRatio - intervalProfits := tradeState.IntervalProfits[types.Interval1d] + tradeStats := sessionTradeStats[session.Name][symbol] - symbolReport, err := createSymbolReport(userConfig, session, symbol, trades.Copy(), intervalProfits, profitFactor, winningRatio) + symbolReport, err := createSymbolReport(userConfig, session, symbol, trades.Copy(), tradeStats) if err != nil { return err } @@ -598,11 +594,12 @@ var BacktestCmd = &cobra.Command{ }, } -func createSymbolReport(userConfig *bbgo.Config, session *bbgo.ExchangeSession, symbol string, trades []types.Trade, intervalProfit *types.IntervalProfitCollector, - profitFactor, winningRatio fixedpoint.Value) ( +func createSymbolReport(userConfig *bbgo.Config, session *bbgo.ExchangeSession, symbol string, trades []types.Trade, tradeStats *types.TradeStats) ( *backtest.SessionSymbolReport, error, ) { + intervalProfit := tradeStats.IntervalProfits[types.Interval1d] + backtestExchange, ok := session.Exchange.(*backtest.Exchange) if !ok { return nil, fmt.Errorf("unexpected error, exchange instance is not a backtest exchange") @@ -612,6 +609,11 @@ func createSymbolReport(userConfig *bbgo.Config, session *bbgo.ExchangeSession, if !ok { return nil, fmt.Errorf("market not found: %s, %s", symbol, session.Exchange.Name()) } + tStart, tEnd := trades[0].Time, trades[len(trades)-1].Time + + periodStart := tStart.Time() + periodEnd := tEnd.Time() + period := periodEnd.Sub(periodStart) startPrice, ok := session.StartPrice(symbol) if !ok { @@ -628,29 +630,74 @@ func createSymbolReport(userConfig *bbgo.Config, session *bbgo.ExchangeSession, Market: market, } - sharpeRatio := fixedpoint.NewFromFloat(intervalProfit.GetSharpe()) - sortinoRatio := fixedpoint.NewFromFloat(intervalProfit.GetSortino()) - report := calculator.Calculate(symbol, trades, lastPrice) accountConfig := userConfig.Backtest.GetAccount(session.Exchange.Name().String()) initBalances := accountConfig.Balances.BalanceMap() finalBalances := session.GetAccount().Balances() + maxProfit := n(intervalProfit.Profits.Max()) + maxLoss := n(math.Abs(intervalProfit.Profits.Min())) + roundTurnCount := n(float64(tradeStats.NumOfProfitTrade + tradeStats.NumOfLossTrade)) + roundTurnLength := n(float64(intervalProfit.Profits.Length())) + winningCount := n(float64(tradeStats.NumOfProfitTrade)) + loosingCount := n(float64(tradeStats.NumOfLossTrade)) + avgProfit := tradeStats.GrossProfit.Div(n(types.NNZ(float64(tradeStats.NumOfProfitTrade), 1))) + avgLoss := tradeStats.GrossLoss.Div(n(types.NNZ(float64(tradeStats.NumOfLossTrade), 1))) + + winningPct := winningCount.Div(roundTurnCount) + // losingPct := fixedpoint.One.Sub(winningPct) + + sharpeRatio := n(intervalProfit.GetSharpe()) + sortinoRatio := n(intervalProfit.GetSortino()) + annVolHis := n(types.AnnualHistoricVolatility(intervalProfit.Profits)) + totalTimeInMarketSec, avgHoldSec := intervalProfit.GetTimeInMarket() + statn, stdErr := types.StatN(*intervalProfit.Profits) symbolReport := backtest.SessionSymbolReport{ - Exchange: session.Exchange.Name(), - Symbol: symbol, - Market: market, - LastPrice: lastPrice, - StartPrice: startPrice, - PnL: report, - InitialBalances: initBalances, - FinalBalances: finalBalances, - // Manifests: manifests, - Sharpe: sharpeRatio, - Sortino: sortinoRatio, - ProfitFactor: profitFactor, - WinningRatio: winningRatio, + Exchange: session.Exchange.Name(), + Symbol: symbol, + Market: market, + LastPrice: lastPrice, + StartPrice: startPrice, + InitialBalances: initBalances, + FinalBalances: finalBalances, + TradeCount: fixedpoint.NewFromInt(int64(len(trades))), + GrossLoss: tradeStats.GrossLoss, + GrossProfit: tradeStats.GrossProfit, + WinningCount: tradeStats.NumOfProfitTrade, + LosingCount: tradeStats.NumOfLossTrade, + RoundTurnCount: roundTurnCount, + WinningRatio: tradeStats.WinningRatio, + PercentProfitable: winningPct, + ProfitFactor: tradeStats.ProfitFactor, + MaxProfit: maxProfit, + MaxLoss: maxLoss, + MaxLossStreak: tradeStats.MaximumConsecutiveLosses, + TotalTimeInMarketSec: totalTimeInMarketSec, + AvgHoldSec: avgHoldSec, + AvgProfit: avgProfit, + AvgLoss: avgLoss, + AvgNetProfit: tradeStats.TotalNetProfit.Div(roundTurnLength), + TotalNetProfit: tradeStats.TotalNetProfit, + AnnualHistoricVolatility: annVolHis, + PnL: report, + PRR: types.PRR(tradeStats.GrossProfit, tradeStats.GrossLoss, winningCount, loosingCount), + Kelly: types.KellyCriterion(tradeStats.ProfitFactor, winningPct), + OptimalF: types.OptimalF(*intervalProfit.Profits), + StatN: statn, + StatNStdErr: stdErr, + Sharpe: sharpeRatio, + Sortino: sortinoRatio, } + cagr := types.NN( + types.CAGR( + symbolReport.InitialEquityValue().Float64(), + symbolReport.FinalEquityValue().Float64(), + int(period.Hours())/24, + ), 0) + + symbolReport.CAGR = n(cagr) + symbolReport.Calmar = n(types.CalmarRatio(cagr, tradeStats.MaxDrawdown)) + for _, s := range session.Subscriptions { symbolReport.Subscriptions = append(symbolReport.Subscriptions, s) } @@ -669,6 +716,9 @@ func createSymbolReport(userConfig *bbgo.Config, session *bbgo.ExchangeSession, return &symbolReport, nil } +func n(v float64) fixedpoint.Value { + return fixedpoint.NewFromFloat(v) +} func verify(userConfig *bbgo.Config, backtestService *service.BacktestService, sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time) error { for _, sourceExchange := range sourceExchanges { err := backtestService.Verify(sourceExchange, userConfig.Backtest.Symbols, startTime, endTime) diff --git a/pkg/types/trade_stat.go b/pkg/types/trade_stat.go new file mode 100644 index 000000000..aee26dbd7 --- /dev/null +++ b/pkg/types/trade_stat.go @@ -0,0 +1,107 @@ +package types + +import ( + "math" + + "gonum.org/v1/gonum/stat" + + "github.com/c9s/bbgo/pkg/datatype/floats" + "github.com/c9s/bbgo/pkg/fixedpoint" +) + +const ( + // DailyToAnnualFactor is the factor to scale daily observations to annual. + // Commonly defined as the number of public market trading days in a year. + DailyToAnnualFactor = 252 // todo does this apply to crypto at all? +) + +// HistVolAnn is the annualized historic volatility of daily returns. +func AnnualHistoricVolatility(data Series) float64 { + var sd = Stdev(data, data.Length(), 1) + return sd * math.Sqrt(DailyToAnnualFactor) +} + +// CAGR Compound Annual Growth Rate +func CAGR(initial, final float64, days int) float64 { + var ( + growthRate = (final - initial) / initial + x = 1 + growthRate + y = 365.0 / float64(days) + ) + return math.Pow(x, y) - 1 +} + +// CalmarRatio relates the capaital growth rate to the maximum drawdown. +func CalmarRatio(cagr, maxDrawdown float64) float64 { + return cagr / maxDrawdown +} + +// KellyCriterion the famous method for trade sizing. +func KellyCriterion(profitFactor, winP fixedpoint.Value) fixedpoint.Value { + return profitFactor.Mul(winP).Sub(fixedpoint.One.Sub(winP)).Div(profitFactor) +} + +// PRR (Pessimistic Return Ratio) is the profit factor with a penalty for a lower number of roundturns. +func PRR(profit, loss, winningN, losingN fixedpoint.Value) fixedpoint.Value { + var ( + winF = 1 / math.Sqrt(1+winningN.Float64()) + loseF = 1 / math.Sqrt(1+losingN.Float64()) + ) + return fixedpoint.NewFromFloat((1 - winF) / (1 + loseF) * (1 + profit.Float64()) / (1 + loss.Float64())) +} + +// StatN returns the statistically significant number of samples required based on the distribution of a series. +// From: https://www.elitetrader.com/et/threads/minimum-number-of-roundturns-required-for-backtesting-results-to-be-trusted.356588/page-2 +func StatN(xs floats.Slice) (sn, se fixedpoint.Value) { + var ( + sd = Stdev(xs, xs.Length(), 1) + m = Mean(xs) + statn = math.Pow(4*(sd/m), 2) + stdErr = stat.StdErr(sd, float64(xs.Length())) + ) + return fixedpoint.NewFromFloat(statn), fixedpoint.NewFromFloat(stdErr) +} + +// OptimalF is a function that returns the 'OptimalF' for a series of trade returns as defined by Ralph Vince. +// It is a method for sizing positions to maximize geometric return whilst accounting for biggest trading loss. +// See: https://www.investopedia.com/terms/o/optimalf.asp +// Param roundturns is the series of profits (-ve amount for losses) for each trade +func OptimalF(roundturns floats.Slice) fixedpoint.Value { + var ( + maxTWR, optimalF float64 + maxLoss = roundturns.Min() + ) + for i := 1.0; i <= 100.0; i++ { + twr := 1.0 + f := i / 100 + for j := range roundturns { + if roundturns[j] == 0 { + continue + } + hpr := 1 + f*(-roundturns[j]/maxLoss) + twr *= hpr + } + if twr > maxTWR { + maxTWR = twr + optimalF = f + } + } + + return fixedpoint.NewFromFloat(optimalF) +} + +// NN (Not Number) returns y if x is NaN or Inf. +func NN(x, y float64) float64 { + if math.IsNaN(x) || math.IsInf(x, 0) { + return y + } + return x +} + +// NNZ (Not Number or Zero) returns y if x is NaN or Inf or Zero. +func NNZ(x, y float64) float64 { + if NN(x, y) == y || x == 0 { + return y + } + return x +} diff --git a/pkg/types/trade_stat_test.go b/pkg/types/trade_stat_test.go new file mode 100644 index 000000000..2b8312b11 --- /dev/null +++ b/pkg/types/trade_stat_test.go @@ -0,0 +1,44 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/datatype/floats" + "github.com/c9s/bbgo/pkg/fixedpoint" +) + +func TestCAGR(t *testing.T) { + giveInitial := 1000.0 + giveFinal := 2500.0 + giveDays := 190 + want := 4.81 + act := CAGR(giveInitial, giveFinal, giveDays) + assert.InDelta(t, want, act, 0.01) +} + +func TestKellyCriterion(t *testing.T) { + var ( + giveProfitFactor = fixedpoint.NewFromFloat(1.6) + giveWinP = fixedpoint.NewFromFloat(0.7) + want = 0.51 + act = KellyCriterion(giveProfitFactor, giveWinP) + ) + assert.InDelta(t, want, act.Float64(), 0.01) +} + +func TestAnnualHistoricVolatility(t *testing.T) { + var ( + give = floats.Slice{0.1, 0.2, -0.15, 0.1, 0.8, -0.3, 0.2} + want = 5.51 + act = AnnualHistoricVolatility(give) + ) + assert.InDelta(t, want, act, 0.01) +} + +func TestOptimalF(t *testing.T) { + roundturns := floats.Slice{10, 20, 50, -10, 40, -40} + f := OptimalF(roundturns) + assert.EqualValues(t, 0.45, f) +} diff --git a/pkg/types/trade_stats.go b/pkg/types/trade_stats.go index b20d3791c..e330fa85b 100644 --- a/pkg/types/trade_stats.go +++ b/pkg/types/trade_stats.go @@ -8,18 +8,23 @@ import ( "time" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" "github.com/c9s/bbgo/pkg/datatype/floats" "github.com/c9s/bbgo/pkg/fixedpoint" ) +const ( + ErrStartTimeNotValid = "No valid start time. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?" + ErrProfitArrEmpty = "profits array empty. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?" +) + type IntervalProfitCollector struct { - Interval Interval `json:"interval"` - Profits *floats.Slice `json:"profits"` - Timestamp *floats.Slice `json:"timestamp"` - tmpTime time.Time `json:"tmpTime"` + Interval Interval `json:"interval"` + Profits *floats.Slice `json:"profits"` + TimeInMarket []time.Duration `json:"timeInMarket"` + Timestamp *floats.Slice `json:"timestamp"` + tmpTime time.Time `json:"tmpTime"` } func NewIntervalProfitCollector(i Interval, startTime time.Time) *IntervalProfitCollector { @@ -28,8 +33,10 @@ func NewIntervalProfitCollector(i Interval, startTime time.Time) *IntervalProfit // Update the collector by every traded profit func (s *IntervalProfitCollector) Update(profit *Profit) { + s.TimeInMarket = append(s.TimeInMarket, profit.TradedAt.Sub(profit.PositionOpenedAt)) + if s.tmpTime.IsZero() { - panic("No valid start time. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?") + panic(ErrStartTimeNotValid) } else { duration := s.Interval.Duration() if profit.TradedAt.Before(s.tmpTime.Add(duration)) { @@ -62,6 +69,20 @@ func (s ProfitReport) String() string { return string(b) } +// Determine average and total time spend in market +func (s *IntervalProfitCollector) GetTimeInMarket() (avgHoldSec, totalTimeInMarketSec int64) { + if s.Profits == nil { + return 0, 0 + } + l := len(s.TimeInMarket) + for i := 0; i < l; i++ { + d := s.TimeInMarket[i] + totalTimeInMarketSec += int64(d / time.Millisecond) + } + avgHoldSec = totalTimeInMarketSec / int64(l) + return +} + // Get all none-profitable intervals func (s *IntervalProfitCollector) GetNonProfitableIntervals() (result []ProfitReport) { if s.Profits == nil { @@ -93,7 +114,7 @@ func (s *IntervalProfitCollector) GetProfitableIntervals() (result []ProfitRepor // Get number of profitable traded intervals func (s *IntervalProfitCollector) GetNumOfProfitableIntervals() (profit int) { if s.Profits == nil { - panic("profits array empty. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?") + panic(ErrProfitArrEmpty) } for _, v := range *s.Profits { if v > 1. { @@ -107,7 +128,7 @@ func (s *IntervalProfitCollector) GetNumOfProfitableIntervals() (profit int) { // (no trade within the interval or pnl = 0 will be also included here) func (s *IntervalProfitCollector) GetNumOfNonProfitableIntervals() (nonprofit int) { if s.Profits == nil { - panic("profits array empty. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?") + panic(ErrProfitArrEmpty) } for _, v := range *s.Profits { if v <= 1. { @@ -121,10 +142,11 @@ func (s *IntervalProfitCollector) GetNumOfNonProfitableIntervals() (nonprofit in // no smart sharpe ON for the calculated result func (s *IntervalProfitCollector) GetSharpe() float64 { if s.tmpTime.IsZero() { - panic("No valid start time. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?") + panic(ErrStartTimeNotValid) } if s.Profits == nil { - panic("profits array empty. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?") + panic(ErrStartTimeNotValid) + } return Sharpe(Sub(s.Profits, 1.), s.Profits.Length(), true, false) } @@ -133,10 +155,10 @@ func (s *IntervalProfitCollector) GetSharpe() float64 { // No risk-free return rate and smart sortino OFF for the calculated result. func (s *IntervalProfitCollector) GetSortino() float64 { if s.tmpTime.IsZero() { - panic("No valid start time. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?") + panic(ErrStartTimeNotValid) } if s.Profits == nil { - panic("profits array empty. Did you create IntervalProfitCollector instance using NewIntervalProfitCollector?") + panic(ErrProfitArrEmpty) } return Sortino(Sub(s.Profits, 1.), 0., s.Profits.Length(), true, false) } @@ -197,6 +219,24 @@ type TradeStats struct { consecutiveSide int consecutiveCounter int consecutiveAmount fixedpoint.Value + + // CAGR is the Compound Annual Growth Rate of the equity curve. + CAGR float64 + + // MaxDrawdown is the maximum percentage drawdown of the equity curve + MaxDrawdown float64 + + // MDDRecovery is the recovery time of the maximum drawdown of the equity curve. + MDDRecovery time.Duration + + // HistVolAnn is the historic volatility of the equity curve as annualized std dev. + HistVolAnn float64 + + // Sharpe is the Sharpe ratio of the equity curve. + Sharpe float64 + + // Calmar is the Calmar ratio of the equity curve. + Calmar float64 } func NewTradeStats(symbol string) *TradeStats {