package bbgo import ( "bytes" "encoding/json" "fmt" "io/ioutil" "reflect" "runtime" "strings" "github.com/pkg/errors" "gopkg.in/yaml.v3" "github.com/c9s/bbgo/pkg/datasource/csvsource" "github.com/c9s/bbgo/pkg/datatype" "github.com/c9s/bbgo/pkg/dynamic" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/service" "github.com/c9s/bbgo/pkg/types" ) // DefaultFeeRate set the fee rate for most cases // BINANCE uses 0.1% for both maker and taker // // for BNB holders, it's 0.075% for both maker and taker // // MAX uses 0.050% for maker and 0.15% for taker var DefaultFeeRate = fixedpoint.NewFromFloat(0.075 * 0.01) type PnLReporterConfig struct { AverageCostBySymbols datatype.StringSlice `json:"averageCostBySymbols" yaml:"averageCostBySymbols"` Of datatype.StringSlice `json:"of" yaml:"of"` When datatype.StringSlice `json:"when" yaml:"when"` } // ExchangeStrategyMount wraps the SingleExchangeStrategy with the ExchangeSession name for mounting type ExchangeStrategyMount struct { // Mounts contains the ExchangeSession name to mount Mounts []string `json:"mounts"` // Strategy is the strategy we loaded from config Strategy SingleExchangeStrategy `json:"strategy"` } func (m *ExchangeStrategyMount) Map() (map[string]interface{}, error) { strategyID := m.Strategy.ID() var params map[string]interface{} out, err := json.Marshal(m.Strategy) if err != nil { return nil, err } if err := json.Unmarshal(out, ¶ms); err != nil { return nil, err } return map[string]interface{}{ "on": m.Mounts, strategyID: params, }, nil } type SlackNotification struct { DefaultChannel string `json:"defaultChannel,omitempty" yaml:"defaultChannel,omitempty"` ErrorChannel string `json:"errorChannel,omitempty" yaml:"errorChannel,omitempty"` } type SlackNotificationRouting struct { Trade string `json:"trade,omitempty" yaml:"trade,omitempty"` Order string `json:"order,omitempty" yaml:"order,omitempty"` SubmitOrder string `json:"submitOrder,omitempty" yaml:"submitOrder,omitempty"` PnL string `json:"pnL,omitempty" yaml:"pnL,omitempty"` } type TelegramNotification struct { Broadcast bool `json:"broadcast" yaml:"broadcast"` } type NotificationSwitches struct { Trade bool `json:"trade" yaml:"trade"` Position bool `json:"position" yaml:"position"` OrderUpdate bool `json:"orderUpdate" yaml:"orderUpdate"` SubmitOrder bool `json:"submitOrder" yaml:"submitOrder"` } type NotificationConfig struct { Slack *SlackNotification `json:"slack,omitempty" yaml:"slack,omitempty"` Telegram *TelegramNotification `json:"telegram,omitempty" yaml:"telegram,omitempty"` Switches *NotificationSwitches `json:"switches" yaml:"switches"` } type LoggingConfig struct { Trade bool `json:"trade,omitempty"` Order bool `json:"order,omitempty"` Balance bool `json:"balance,omitempty"` FilledOrderOnly bool `json:"filledOrder,omitempty"` Fields map[string]interface{} `json:"fields,omitempty"` } type Session struct { Name string `json:"name,omitempty" yaml:"name,omitempty"` ExchangeName string `json:"exchange" yaml:"exchange"` EnvVarPrefix string `json:"envVarPrefix" yaml:"envVarPrefix"` Key string `json:"key,omitempty" yaml:"key,omitempty"` Secret string `json:"secret,omitempty" yaml:"secret,omitempty"` PublicOnly bool `json:"publicOnly,omitempty" yaml:"publicOnly"` Margin bool `json:"margin,omitempty" yaml:"margin,omitempty"` IsolatedMargin bool `json:"isolatedMargin,omitempty" yaml:"isolatedMargin,omitempty"` IsolatedMarginSymbol string `json:"isolatedMarginSymbol,omitempty" yaml:"isolatedMarginSymbol,omitempty"` } //go:generate go run github.com/dmarkham/enumer -type=BacktestFeeMode -transform=snake -trimprefix BacktestFeeMode -yaml -json type BacktestFeeMode int const ( // BackTestFeeModeQuoteFee is designed for clean position but which also counts the fee in the quote balance. // buy order = quote currency fee // sell order = quote currency fee BacktestFeeModeQuote BacktestFeeMode = iota // quote // BackTestFeeModeNativeFee is the default crypto exchange fee mode. // buy order = base currency fee // sell order = quote currency fee BacktestFeeModeNative // BackTestFeeMode = "native" // BackTestFeeModeFeeToken is the mode which calculates fee from the outside of the balances. // the fee will not be included in the balances nor the profit. BacktestFeeModeToken // BackTestFeeMode = "token" ) type Backtest struct { StartTime types.LooseFormatTime `json:"startTime,omitempty" yaml:"startTime,omitempty"` EndTime *types.LooseFormatTime `json:"endTime,omitempty" yaml:"endTime,omitempty"` // RecordTrades is an option, if set to true, back-testing should record the trades into database RecordTrades bool `json:"recordTrades,omitempty" yaml:"recordTrades,omitempty"` // Deprecated: // Account is deprecated, use Accounts instead Account map[string]BacktestAccount `json:"account" yaml:"account"` FeeMode BacktestFeeMode `json:"feeMode" yaml:"feeMode"` Accounts map[string]BacktestAccount `json:"accounts" yaml:"accounts"` Symbols []string `json:"symbols" yaml:"symbols"` Sessions []string `json:"sessions" yaml:"sessions"` // sync 1 second interval KLines SyncSecKLines bool `json:"syncSecKLines,omitempty" yaml:"syncSecKLines,omitempty"` CsvSource *csvsource.CsvConfig `json:"csvConfig,omitempty" yaml:"csvConfig,omitempty"` } func (b *Backtest) GetAccount(n string) BacktestAccount { accountConfig, ok := b.Accounts[n] if ok { return accountConfig } accountConfig, ok = b.Account[n] if ok { return accountConfig } return DefaultBacktestAccount } type BacktestAccount struct { MakerFeeRate fixedpoint.Value `json:"makerFeeRate,omitempty" yaml:"makerFeeRate,omitempty"` TakerFeeRate fixedpoint.Value `json:"takerFeeRate,omitempty" yaml:"takerFeeRate,omitempty"` Balances BacktestAccountBalanceMap `json:"balances" yaml:"balances"` } var DefaultBacktestAccount = BacktestAccount{ MakerFeeRate: fixedpoint.MustNewFromString("0.050%"), TakerFeeRate: fixedpoint.MustNewFromString("0.075%"), Balances: BacktestAccountBalanceMap{ "USDT": fixedpoint.NewFromFloat(10000), }, } type BA BacktestAccount func (b *BacktestAccount) UnmarshalYAML(value *yaml.Node) error { bb := &BA{MakerFeeRate: DefaultFeeRate, TakerFeeRate: DefaultFeeRate} if err := value.Decode(bb); err != nil { return err } *b = BacktestAccount(*bb) return nil } func (b *BacktestAccount) UnmarshalJSON(input []byte) error { bb := &BA{MakerFeeRate: DefaultFeeRate, TakerFeeRate: DefaultFeeRate} if err := json.Unmarshal(input, bb); err != nil { return err } *b = BacktestAccount(*bb) return nil } type BacktestAccountBalanceMap map[string]fixedpoint.Value func (m BacktestAccountBalanceMap) BalanceMap() types.BalanceMap { balances := make(types.BalanceMap) for currency, value := range m { balances[currency] = types.Balance{ Currency: currency, Available: value, Locked: fixedpoint.Zero, } } return balances } type PersistenceConfig struct { Redis *service.RedisPersistenceConfig `json:"redis,omitempty" yaml:"redis,omitempty"` Json *service.JsonPersistenceConfig `json:"json,omitempty" yaml:"json,omitempty"` } type BuildTargetConfig struct { Name string `json:"name" yaml:"name"` Arch string `json:"arch" yaml:"arch"` OS string `json:"os" yaml:"os"` LDFlags datatype.StringSlice `json:"ldflags,omitempty" yaml:"ldflags,omitempty"` GCFlags datatype.StringSlice `json:"gcflags,omitempty" yaml:"gcflags,omitempty"` Imports []string `json:"imports,omitempty" yaml:"imports,omitempty"` } type BuildConfig struct { BuildDir string `json:"buildDir,omitempty" yaml:"buildDir,omitempty"` Imports []string `json:"imports,omitempty" yaml:"imports,omitempty"` Targets []BuildTargetConfig `json:"targets,omitempty" yaml:"targets,omitempty"` } func GetNativeBuildTargetConfig() BuildTargetConfig { return BuildTargetConfig{ Name: "bbgow", Arch: runtime.GOARCH, OS: runtime.GOOS, } } type SyncSymbol struct { Symbol string `json:"symbol" yaml:"symbol"` Session string `json:"session" yaml:"session"` } func (ss *SyncSymbol) UnmarshalYAML(unmarshal func(a interface{}) error) (err error) { var s string if err = unmarshal(&s); err == nil { aa := strings.SplitN(s, ":", 2) if len(aa) > 1 { ss.Session = aa[0] ss.Symbol = aa[1] } else { ss.Symbol = aa[0] } return nil } type localSyncSymbol SyncSymbol var ssNew localSyncSymbol if err = unmarshal(&ssNew); err == nil { *ss = SyncSymbol(ssNew) return nil } return err } func categorizeSyncSymbol(slice []SyncSymbol) (map[string][]string, []string) { var rest []string var m = make(map[string][]string) for _, ss := range slice { if len(ss.Session) > 0 { m[ss.Session] = append(m[ss.Session], ss.Symbol) } else { rest = append(rest, ss.Symbol) } } return m, rest } type SyncConfig struct { // Sessions to sync, if ignored, all defined sessions will sync Sessions []string `json:"sessions,omitempty" yaml:"sessions,omitempty"` // Symbols is the list of session:symbol pair to sync, if ignored, symbols wlll be discovered by your existing crypto balances // Valid formats are: {session}:{symbol}, {symbol} or in YAML object form {symbol: "BTCUSDT", session:"max" } Symbols []SyncSymbol `json:"symbols,omitempty" yaml:"symbols,omitempty"` // DepositHistory is for syncing deposit history DepositHistory bool `json:"depositHistory" yaml:"depositHistory"` // WithdrawHistory is for syncing withdraw history WithdrawHistory bool `json:"withdrawHistory" yaml:"withdrawHistory"` // RewardHistory is for syncing reward history RewardHistory bool `json:"rewardHistory" yaml:"rewardHistory"` // MarginHistory is for syncing margin related history: loans, repays, interests and liquidations MarginHistory bool `json:"marginHistory" yaml:"marginHistory"` MarginAssets []string `json:"marginAssets" yaml:"marginAssets"` // Since is the date where you want to start syncing data Since *types.LooseFormatTime `json:"since,omitempty"` // UserDataStream is for real-time sync with websocket user data stream UserDataStream *struct { Trades bool `json:"trades,omitempty" yaml:"trades,omitempty"` FilledOrders bool `json:"filledOrders,omitempty" yaml:"filledOrders,omitempty"` } `json:"userDataStream,omitempty" yaml:"userDataStream,omitempty"` } type GoogleSpreadSheetServiceConfig struct { JsonTokenFile string `json:"jsonTokenFile" yaml:"jsonTokenFile"` SpreadSheetID string `json:"spreadSheetId" yaml:"spreadSheetId"` } type ServiceConfig struct { GoogleSpreadSheetService *GoogleSpreadSheetServiceConfig `json:"googleSpreadSheet" yaml:"googleSpreadSheet"` } type DatabaseConfig struct { Driver string `json:"driver" yaml:"driver"` DSN string `json:"dsn" yaml:"dsn"` ExtraMigrationPackages []string `json:"extraMigrationPackages" yaml:"extraMigrationPackages"` } type EnvironmentConfig struct { DisableDefaultKLineSubscription bool `json:"disableDefaultKLineSubscription"` DisableHistoryKLinePreload bool `json:"disableHistoryKLinePreload"` // DisableStartUpBalanceQuery disables the balance query in the startup process // which initializes the session.Account with the QueryAccount method. DisableStartupBalanceQuery bool `json:"disableStartupBalanceQuery"` DisableSessionTradeBuffer bool `json:"disableSessionTradeBuffer"` DisableMarketDataStore bool `json:"disableMarketDataStore"` MaxSessionTradeBufferSize int `json:"maxSessionTradeBufferSize"` SyncBufferPeriod *types.Duration `json:"syncBufferPeriod"` } type Config struct { Build *BuildConfig `json:"build,omitempty" yaml:"build,omitempty"` // Imports is deprecated // Deprecated: use BuildConfig instead Imports []string `json:"imports,omitempty" yaml:"imports,omitempty"` Backtest *Backtest `json:"backtest,omitempty" yaml:"backtest,omitempty"` Sync *SyncConfig `json:"sync,omitempty" yaml:"sync,omitempty"` Notifications *NotificationConfig `json:"notifications,omitempty" yaml:"notifications,omitempty"` Persistence *PersistenceConfig `json:"persistence,omitempty" yaml:"persistence,omitempty"` Service *ServiceConfig `json:"services,omitempty" yaml:"services,omitempty"` DatabaseConfig *DatabaseConfig `json:"database,omitempty" yaml:"database,omitempty"` Environment *EnvironmentConfig `json:"environment,omitempty" yaml:"environment,omitempty"` Sessions map[string]*ExchangeSession `json:"sessions,omitempty" yaml:"sessions,omitempty"` RiskControls *RiskControls `json:"riskControls,omitempty" yaml:"riskControls,omitempty"` Logging *LoggingConfig `json:"logging,omitempty"` ExchangeStrategies []ExchangeStrategyMount `json:"-" yaml:"-"` CrossExchangeStrategies []CrossExchangeStrategy `json:"-" yaml:"-"` PnLReporters []PnLReporterConfig `json:"reportPnL,omitempty" yaml:"reportPnL,omitempty"` } func (c *Config) Map() (map[string]interface{}, error) { text, err := json.Marshal(c) if err != nil { return nil, err } var data map[string]interface{} err = json.Unmarshal(text, &data) if err != nil { return nil, err } // convert strategy config back to the DSL format var exchangeStrategies []map[string]interface{} for _, m := range c.ExchangeStrategies { params, err := m.Map() if err != nil { return nil, err } exchangeStrategies = append(exchangeStrategies, params) } if len(exchangeStrategies) > 0 { data["exchangeStrategies"] = exchangeStrategies } var crossExchangeStrategies []map[string]interface{} for _, st := range c.CrossExchangeStrategies { strategyID := st.ID() var params Stash out, err := json.Marshal(st) if err != nil { return nil, err } if err := json.Unmarshal(out, ¶ms); err != nil { return nil, err } crossExchangeStrategies = append(crossExchangeStrategies, map[string]interface{}{ strategyID: params, }) } if len(crossExchangeStrategies) > 0 { data["crossExchangeStrategies"] = crossExchangeStrategies } return data, err } func (c *Config) YAML() ([]byte, error) { m, err := c.Map() if err != nil { return nil, err } var buf bytes.Buffer var enc = yaml.NewEncoder(&buf) enc.SetIndent(2) err = enc.Encode(m) return buf.Bytes(), err } func (c *Config) GetSignature() string { var s string var ps []string // for single exchange strategy if len(c.ExchangeStrategies) == 1 && len(c.CrossExchangeStrategies) == 0 { mount := c.ExchangeStrategies[0].Mounts[0] ps = append(ps, mount) strategy := c.ExchangeStrategies[0].Strategy id := strategy.ID() ps = append(ps, id) if symbol, ok := dynamic.LookupSymbolField(reflect.ValueOf(strategy)); ok { ps = append(ps, symbol) } } startTime := c.Backtest.StartTime.Time() ps = append(ps, startTime.Format("2006-01-02")) if c.Backtest.EndTime != nil { endTime := c.Backtest.EndTime.Time() ps = append(ps, endTime.Format("2006-01-02")) } s = strings.Join(ps, "_") return s } type Stash map[string]interface{} func loadStash(config []byte) (Stash, error) { stash := make(Stash) if err := yaml.Unmarshal(config, stash); err != nil { return nil, err } return stash, nil } func LoadBuildConfig(configFile string) (*Config, error) { var config Config content, err := ioutil.ReadFile(configFile) if err != nil { return nil, err } if err := yaml.Unmarshal(content, &config); err != nil { return nil, err } // for backward compatible if config.Build == nil { if len(config.Imports) > 0 { config.Build = &BuildConfig{ BuildDir: "build", Imports: config.Imports, Targets: []BuildTargetConfig{ {Name: "bbgow-amd64-darwin", Arch: "amd64", OS: "darwin"}, {Name: "bbgow-amd64-linux", Arch: "amd64", OS: "linux"}, }, } } } return &config, nil } // Load parses the config func Load(configFile string, loadStrategies bool) (*Config, error) { var config Config content, err := ioutil.ReadFile(configFile) if err != nil { return nil, err } if err := yaml.Unmarshal(content, &config); err != nil { return nil, err } // for backward compatible if config.Build == nil { config.Build = &BuildConfig{ BuildDir: "build", Imports: config.Imports, Targets: []BuildTargetConfig{ {Name: "bbgow-amd64-darwin", Arch: "amd64", OS: "darwin"}, {Name: "bbgow-amd64-linux", Arch: "amd64", OS: "linux"}, }, } } stash, err := loadStash(content) if err != nil { return nil, err } if loadStrategies { if err := loadExchangeStrategies(&config, stash); err != nil { return nil, err } if err := loadCrossExchangeStrategies(&config, stash); err != nil { return nil, err } } return &config, nil } func loadCrossExchangeStrategies(config *Config, stash Stash) (err error) { exchangeStrategiesConf, ok := stash["crossExchangeStrategies"] if !ok { return nil } if len(LoadedCrossExchangeStrategies) == 0 { return errors.New("no cross exchange strategy is registered") } configList, ok := exchangeStrategiesConf.([]interface{}) if !ok { return errors.New("expecting list in crossExchangeStrategies") } for _, entry := range configList { configStash, ok := entry.(Stash) if !ok { return fmt.Errorf("strategy config should be a map, given: %T %+v", entry, entry) } for id, conf := range configStash { // look up the real struct type if st, ok := LoadedCrossExchangeStrategies[id]; ok { val, err := reUnmarshal(conf, st) if err != nil { return err } config.CrossExchangeStrategies = append(config.CrossExchangeStrategies, val.(CrossExchangeStrategy)) } } } return nil } func NewStrategyFromMap(id string, conf interface{}) (SingleExchangeStrategy, error) { if st, ok := LoadedExchangeStrategies[id]; ok { val, err := reUnmarshal(conf, st) if err != nil { return nil, err } return val.(SingleExchangeStrategy), nil } return nil, fmt.Errorf("strategy %s not found", id) } func loadExchangeStrategies(config *Config, stash Stash) (err error) { exchangeStrategiesConf, ok := stash["exchangeStrategies"] if !ok { exchangeStrategiesConf, ok = stash["strategies"] if !ok { return nil } } if len(LoadedExchangeStrategies) == 0 { return errors.New("no exchange strategy is registered") } configList, ok := exchangeStrategiesConf.([]interface{}) if !ok { return errors.New("expecting list in exchangeStrategies") } for _, entry := range configList { configStash, ok := entry.(Stash) if !ok { return fmt.Errorf("strategy config should be a map, given: %T %+v", entry, entry) } var mounts []string if val, ok := configStash["on"]; ok { switch tv := val.(type) { case []string: mounts = append(mounts, tv...) case string: mounts = append(mounts, tv) case []interface{}: for _, f := range tv { s, ok := f.(string) if !ok { return fmt.Errorf("%+v (%T) is not a string", f, f) } mounts = append(mounts, s) } default: return fmt.Errorf("unexpected mount type: %T value: %+v", val, val) } } for id, conf := range configStash { // look up the real struct type if _, ok := LoadedExchangeStrategies[id]; ok { st, err := NewStrategyFromMap(id, conf) if err != nil { return err } config.ExchangeStrategies = append(config.ExchangeStrategies, ExchangeStrategyMount{ Mounts: mounts, Strategy: st, }) } else if id != "on" && id != "off" { // Show error when we didn't find the Strategy return fmt.Errorf("strategy %s in config not found", id) } } } return nil } func reUnmarshal(conf interface{}, tpe interface{}) (interface{}, error) { // get the type "*Strategy" rt := reflect.TypeOf(tpe) // allocate new object from the given type val := reflect.New(rt) // now we have &(*Strategy) -> **Strategy valRef := val.Interface() plain, err := json.Marshal(conf) if err != nil { return nil, err } if err := json.Unmarshal(plain, valRef); err != nil { return nil, errors.Wrapf(err, "json parsing error, given payload: %s", plain) } return val.Elem().Interface(), nil }