bbgo_origin/pkg/bbgo/config.go

695 lines
19 KiB
Go

package bbgo
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"reflect"
"runtime"
"strings"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
"github.com/c9s/bbgo/pkg/datatype"
"github.com/c9s/bbgo/pkg/dynamic"
"github.com/c9s/bbgo/pkg/fixedpoint"
"github.com/c9s/bbgo/pkg/service"
"github.com/c9s/bbgo/pkg/types"
)
// DefaultFeeRate set the fee rate for most cases
// BINANCE uses 0.1% for both maker and taker
//
// for BNB holders, it's 0.075% for both maker and taker
//
// MAX uses 0.050% for maker and 0.15% for taker
var DefaultFeeRate = fixedpoint.NewFromFloat(0.075 * 0.01)
type PnLReporterConfig struct {
AverageCostBySymbols datatype.StringSlice `json:"averageCostBySymbols" yaml:"averageCostBySymbols"`
Of datatype.StringSlice `json:"of" yaml:"of"`
When datatype.StringSlice `json:"when" yaml:"when"`
}
// ExchangeStrategyMount wraps the SingleExchangeStrategy with the ExchangeSession name for mounting
type ExchangeStrategyMount struct {
// Mounts contains the ExchangeSession name to mount
Mounts []string `json:"mounts"`
// Strategy is the strategy we loaded from config
Strategy SingleExchangeStrategy `json:"strategy"`
}
func (m *ExchangeStrategyMount) Map() (map[string]interface{}, error) {
strategyID := m.Strategy.ID()
var params map[string]interface{}
out, err := json.Marshal(m.Strategy)
if err != nil {
return nil, err
}
if err := json.Unmarshal(out, &params); err != nil {
return nil, err
}
return map[string]interface{}{
"on": m.Mounts,
strategyID: params,
}, nil
}
type SlackNotification struct {
DefaultChannel string `json:"defaultChannel,omitempty" yaml:"defaultChannel,omitempty"`
ErrorChannel string `json:"errorChannel,omitempty" yaml:"errorChannel,omitempty"`
}
type SlackNotificationRouting struct {
Trade string `json:"trade,omitempty" yaml:"trade,omitempty"`
Order string `json:"order,omitempty" yaml:"order,omitempty"`
SubmitOrder string `json:"submitOrder,omitempty" yaml:"submitOrder,omitempty"`
PnL string `json:"pnL,omitempty" yaml:"pnL,omitempty"`
}
type TelegramNotification struct {
Broadcast bool `json:"broadcast" yaml:"broadcast"`
}
type NotificationSwitches struct {
Trade bool `json:"trade" yaml:"trade"`
Position bool `json:"position" yaml:"position"`
OrderUpdate bool `json:"orderUpdate" yaml:"orderUpdate"`
SubmitOrder bool `json:"submitOrder" yaml:"submitOrder"`
}
type NotificationConfig struct {
Slack *SlackNotification `json:"slack,omitempty" yaml:"slack,omitempty"`
Telegram *TelegramNotification `json:"telegram,omitempty" yaml:"telegram,omitempty"`
Switches *NotificationSwitches `json:"switches" yaml:"switches"`
}
type LoggingConfig struct {
Trade bool `json:"trade,omitempty"`
Order bool `json:"order,omitempty"`
Balance bool `json:"balance,omitempty"`
FilledOrderOnly bool `json:"filledOrder,omitempty"`
Fields map[string]interface{} `json:"fields,omitempty"`
}
type Session struct {
Name string `json:"name,omitempty" yaml:"name,omitempty"`
ExchangeName string `json:"exchange" yaml:"exchange"`
EnvVarPrefix string `json:"envVarPrefix" yaml:"envVarPrefix"`
Key string `json:"key,omitempty" yaml:"key,omitempty"`
Secret string `json:"secret,omitempty" yaml:"secret,omitempty"`
PublicOnly bool `json:"publicOnly,omitempty" yaml:"publicOnly"`
Margin bool `json:"margin,omitempty" yaml:"margin,omitempty"`
IsolatedMargin bool `json:"isolatedMargin,omitempty" yaml:"isolatedMargin,omitempty"`
IsolatedMarginSymbol string `json:"isolatedMarginSymbol,omitempty" yaml:"isolatedMarginSymbol,omitempty"`
}
//go:generate go run github.com/dmarkham/enumer -type=BacktestFeeMode -transform=snake -trimprefix BacktestFeeMode -yaml -json
type BacktestFeeMode int
const (
// BackTestFeeModeQuoteFee is designed for clean position but which also counts the fee in the quote balance.
// buy order = quote currency fee
// sell order = quote currency fee
BacktestFeeModeQuote BacktestFeeMode = iota // quote
// BackTestFeeModeNativeFee is the default crypto exchange fee mode.
// buy order = base currency fee
// sell order = quote currency fee
BacktestFeeModeNative // BackTestFeeMode = "native"
// BackTestFeeModeFeeToken is the mode which calculates fee from the outside of the balances.
// the fee will not be included in the balances nor the profit.
BacktestFeeModeToken // BackTestFeeMode = "token"
)
type Backtest struct {
StartTime types.LooseFormatTime `json:"startTime,omitempty" yaml:"startTime,omitempty"`
EndTime *types.LooseFormatTime `json:"endTime,omitempty" yaml:"endTime,omitempty"`
// RecordTrades is an option, if set to true, back-testing should record the trades into database
RecordTrades bool `json:"recordTrades,omitempty" yaml:"recordTrades,omitempty"`
// Deprecated:
// Account is deprecated, use Accounts instead
Account map[string]BacktestAccount `json:"account" yaml:"account"`
FeeMode BacktestFeeMode `json:"feeMode" yaml:"feeMode"`
Accounts map[string]BacktestAccount `json:"accounts" yaml:"accounts"`
Symbols []string `json:"symbols" yaml:"symbols"`
Sessions []string `json:"sessions" yaml:"sessions"`
// sync 1 second interval KLines
SyncSecKLines bool `json:"syncSecKLines,omitempty" yaml:"syncSecKLines,omitempty"`
}
func (b *Backtest) GetAccount(n string) BacktestAccount {
accountConfig, ok := b.Accounts[n]
if ok {
return accountConfig
}
accountConfig, ok = b.Account[n]
if ok {
return accountConfig
}
return DefaultBacktestAccount
}
type BacktestAccount struct {
MakerFeeRate fixedpoint.Value `json:"makerFeeRate,omitempty" yaml:"makerFeeRate,omitempty"`
TakerFeeRate fixedpoint.Value `json:"takerFeeRate,omitempty" yaml:"takerFeeRate,omitempty"`
Balances BacktestAccountBalanceMap `json:"balances" yaml:"balances"`
}
var DefaultBacktestAccount = BacktestAccount{
MakerFeeRate: fixedpoint.MustNewFromString("0.050%"),
TakerFeeRate: fixedpoint.MustNewFromString("0.075%"),
Balances: BacktestAccountBalanceMap{
"USDT": fixedpoint.NewFromFloat(10000),
},
}
type BA BacktestAccount
func (b *BacktestAccount) UnmarshalYAML(value *yaml.Node) error {
bb := &BA{MakerFeeRate: DefaultFeeRate, TakerFeeRate: DefaultFeeRate}
if err := value.Decode(bb); err != nil {
return err
}
*b = BacktestAccount(*bb)
return nil
}
func (b *BacktestAccount) UnmarshalJSON(input []byte) error {
bb := &BA{MakerFeeRate: DefaultFeeRate, TakerFeeRate: DefaultFeeRate}
if err := json.Unmarshal(input, bb); err != nil {
return err
}
*b = BacktestAccount(*bb)
return nil
}
type BacktestAccountBalanceMap map[string]fixedpoint.Value
func (m BacktestAccountBalanceMap) BalanceMap() types.BalanceMap {
balances := make(types.BalanceMap)
for currency, value := range m {
balances[currency] = types.Balance{
Currency: currency,
Available: value,
Locked: fixedpoint.Zero,
}
}
return balances
}
type PersistenceConfig struct {
Redis *service.RedisPersistenceConfig `json:"redis,omitempty" yaml:"redis,omitempty"`
Json *service.JsonPersistenceConfig `json:"json,omitempty" yaml:"json,omitempty"`
}
type BuildTargetConfig struct {
Name string `json:"name" yaml:"name"`
Arch string `json:"arch" yaml:"arch"`
OS string `json:"os" yaml:"os"`
LDFlags datatype.StringSlice `json:"ldflags,omitempty" yaml:"ldflags,omitempty"`
GCFlags datatype.StringSlice `json:"gcflags,omitempty" yaml:"gcflags,omitempty"`
Imports []string `json:"imports,omitempty" yaml:"imports,omitempty"`
}
type BuildConfig struct {
BuildDir string `json:"buildDir,omitempty" yaml:"buildDir,omitempty"`
Imports []string `json:"imports,omitempty" yaml:"imports,omitempty"`
Targets []BuildTargetConfig `json:"targets,omitempty" yaml:"targets,omitempty"`
}
func GetNativeBuildTargetConfig() BuildTargetConfig {
return BuildTargetConfig{
Name: "bbgow",
Arch: runtime.GOARCH,
OS: runtime.GOOS,
}
}
type SyncSymbol struct {
Symbol string `json:"symbol" yaml:"symbol"`
Session string `json:"session" yaml:"session"`
}
func (ss *SyncSymbol) UnmarshalYAML(unmarshal func(a interface{}) error) (err error) {
var s string
if err = unmarshal(&s); err == nil {
aa := strings.SplitN(s, ":", 2)
if len(aa) > 1 {
ss.Session = aa[0]
ss.Symbol = aa[1]
} else {
ss.Symbol = aa[0]
}
return nil
}
type localSyncSymbol SyncSymbol
var ssNew localSyncSymbol
if err = unmarshal(&ssNew); err == nil {
*ss = SyncSymbol(ssNew)
return nil
}
return err
}
func categorizeSyncSymbol(slice []SyncSymbol) (map[string][]string, []string) {
var rest []string
var m = make(map[string][]string)
for _, ss := range slice {
if len(ss.Session) > 0 {
m[ss.Session] = append(m[ss.Session], ss.Symbol)
} else {
rest = append(rest, ss.Symbol)
}
}
return m, rest
}
type SyncConfig struct {
// Sessions to sync, if ignored, all defined sessions will sync
Sessions []string `json:"sessions,omitempty" yaml:"sessions,omitempty"`
// Symbols is the list of session:symbol pair to sync, if ignored, symbols wlll be discovered by your existing crypto balances
// Valid formats are: {session}:{symbol}, {symbol} or in YAML object form {symbol: "BTCUSDT", session:"max" }
Symbols []SyncSymbol `json:"symbols,omitempty" yaml:"symbols,omitempty"`
// DepositHistory is for syncing deposit history
DepositHistory bool `json:"depositHistory" yaml:"depositHistory"`
// WithdrawHistory is for syncing withdraw history
WithdrawHistory bool `json:"withdrawHistory" yaml:"withdrawHistory"`
// RewardHistory is for syncing reward history
RewardHistory bool `json:"rewardHistory" yaml:"rewardHistory"`
// MarginHistory is for syncing margin related history: loans, repays, interests and liquidations
MarginHistory bool `json:"marginHistory" yaml:"marginHistory"`
MarginAssets []string `json:"marginAssets" yaml:"marginAssets"`
// Since is the date where you want to start syncing data
Since *types.LooseFormatTime `json:"since,omitempty"`
// UserDataStream is for real-time sync with websocket user data stream
UserDataStream *struct {
Trades bool `json:"trades,omitempty" yaml:"trades,omitempty"`
FilledOrders bool `json:"filledOrders,omitempty" yaml:"filledOrders,omitempty"`
} `json:"userDataStream,omitempty" yaml:"userDataStream,omitempty"`
}
type GoogleSpreadSheetServiceConfig struct {
JsonTokenFile string `json:"jsonTokenFile" yaml:"jsonTokenFile"`
SpreadSheetID string `json:"spreadSheetId" yaml:"spreadSheetId"`
}
type ServiceConfig struct {
GoogleSpreadSheetService *GoogleSpreadSheetServiceConfig `json:"googleSpreadSheet" yaml:"googleSpreadSheet"`
}
type EnvironmentConfig struct {
DisableDefaultKLineSubscription bool `json:"disableDefaultKLineSubscription"`
DisableHistoryKLinePreload bool `json:"disableHistoryKLinePreload"`
// DisableStartUpBalanceQuery disables the balance query in the startup process
// which initializes the session.Account with the QueryAccount method.
DisableStartupBalanceQuery bool `json:"disableStartupBalanceQuery"`
DisableSessionTradeBuffer bool `json:"disableSessionTradeBuffer"`
MaxSessionTradeBufferSize int `json:"maxSessionTradeBufferSize"`
}
type Config struct {
Build *BuildConfig `json:"build,omitempty" yaml:"build,omitempty"`
// Imports is deprecated
// Deprecated: use BuildConfig instead
Imports []string `json:"imports,omitempty" yaml:"imports,omitempty"`
Backtest *Backtest `json:"backtest,omitempty" yaml:"backtest,omitempty"`
Sync *SyncConfig `json:"sync,omitempty" yaml:"sync,omitempty"`
Notifications *NotificationConfig `json:"notifications,omitempty" yaml:"notifications,omitempty"`
Persistence *PersistenceConfig `json:"persistence,omitempty" yaml:"persistence,omitempty"`
Service *ServiceConfig `json:"services,omitempty" yaml:"services,omitempty"`
Environment *EnvironmentConfig `json:"environment,omitempty" yaml:"environment,omitempty"`
Sessions map[string]*ExchangeSession `json:"sessions,omitempty" yaml:"sessions,omitempty"`
RiskControls *RiskControls `json:"riskControls,omitempty" yaml:"riskControls,omitempty"`
Logging *LoggingConfig `json:"logging,omitempty"`
ExchangeStrategies []ExchangeStrategyMount `json:"-" yaml:"-"`
CrossExchangeStrategies []CrossExchangeStrategy `json:"-" yaml:"-"`
PnLReporters []PnLReporterConfig `json:"reportPnL,omitempty" yaml:"reportPnL,omitempty"`
}
func (c *Config) Map() (map[string]interface{}, error) {
text, err := json.Marshal(c)
if err != nil {
return nil, err
}
var data map[string]interface{}
err = json.Unmarshal(text, &data)
if err != nil {
return nil, err
}
// convert strategy config back to the DSL format
var exchangeStrategies []map[string]interface{}
for _, m := range c.ExchangeStrategies {
params, err := m.Map()
if err != nil {
return nil, err
}
exchangeStrategies = append(exchangeStrategies, params)
}
if len(exchangeStrategies) > 0 {
data["exchangeStrategies"] = exchangeStrategies
}
var crossExchangeStrategies []map[string]interface{}
for _, st := range c.CrossExchangeStrategies {
strategyID := st.ID()
var params Stash
out, err := json.Marshal(st)
if err != nil {
return nil, err
}
if err := json.Unmarshal(out, &params); err != nil {
return nil, err
}
crossExchangeStrategies = append(crossExchangeStrategies, map[string]interface{}{
strategyID: params,
})
}
if len(crossExchangeStrategies) > 0 {
data["crossExchangeStrategies"] = crossExchangeStrategies
}
return data, err
}
func (c *Config) YAML() ([]byte, error) {
m, err := c.Map()
if err != nil {
return nil, err
}
var buf bytes.Buffer
var enc = yaml.NewEncoder(&buf)
enc.SetIndent(2)
err = enc.Encode(m)
return buf.Bytes(), err
}
func (c *Config) GetSignature() string {
var s string
var ps []string
// for single exchange strategy
if len(c.ExchangeStrategies) == 1 && len(c.CrossExchangeStrategies) == 0 {
mount := c.ExchangeStrategies[0].Mounts[0]
ps = append(ps, mount)
strategy := c.ExchangeStrategies[0].Strategy
id := strategy.ID()
ps = append(ps, id)
if symbol, ok := dynamic.LookupSymbolField(reflect.ValueOf(strategy)); ok {
ps = append(ps, symbol)
}
}
startTime := c.Backtest.StartTime.Time()
ps = append(ps, startTime.Format("2006-01-02"))
if c.Backtest.EndTime != nil {
endTime := c.Backtest.EndTime.Time()
ps = append(ps, endTime.Format("2006-01-02"))
}
s = strings.Join(ps, "_")
return s
}
type Stash map[string]interface{}
func loadStash(config []byte) (Stash, error) {
stash := make(Stash)
if err := yaml.Unmarshal(config, stash); err != nil {
return nil, err
}
return stash, nil
}
func LoadBuildConfig(configFile string) (*Config, error) {
var config Config
content, err := ioutil.ReadFile(configFile)
if err != nil {
return nil, err
}
if err := yaml.Unmarshal(content, &config); err != nil {
return nil, err
}
// for backward compatible
if config.Build == nil {
if len(config.Imports) > 0 {
config.Build = &BuildConfig{
BuildDir: "build",
Imports: config.Imports,
Targets: []BuildTargetConfig{
{Name: "bbgow-amd64-darwin", Arch: "amd64", OS: "darwin"},
{Name: "bbgow-amd64-linux", Arch: "amd64", OS: "linux"},
},
}
}
}
return &config, nil
}
// Load parses the config
func Load(configFile string, loadStrategies bool) (*Config, error) {
var config Config
content, err := ioutil.ReadFile(configFile)
if err != nil {
return nil, err
}
if err := yaml.Unmarshal(content, &config); err != nil {
return nil, err
}
// for backward compatible
if config.Build == nil {
config.Build = &BuildConfig{
BuildDir: "build",
Imports: config.Imports,
Targets: []BuildTargetConfig{
{Name: "bbgow-amd64-darwin", Arch: "amd64", OS: "darwin"},
{Name: "bbgow-amd64-linux", Arch: "amd64", OS: "linux"},
},
}
}
stash, err := loadStash(content)
if err != nil {
return nil, err
}
if loadStrategies {
if err := loadExchangeStrategies(&config, stash); err != nil {
return nil, err
}
if err := loadCrossExchangeStrategies(&config, stash); err != nil {
return nil, err
}
}
return &config, nil
}
func loadCrossExchangeStrategies(config *Config, stash Stash) (err error) {
exchangeStrategiesConf, ok := stash["crossExchangeStrategies"]
if !ok {
return nil
}
if len(LoadedCrossExchangeStrategies) == 0 {
return errors.New("no cross exchange strategy is registered")
}
configList, ok := exchangeStrategiesConf.([]interface{})
if !ok {
return errors.New("expecting list in crossExchangeStrategies")
}
for _, entry := range configList {
configStash, ok := entry.(Stash)
if !ok {
return fmt.Errorf("strategy config should be a map, given: %T %+v", entry, entry)
}
for id, conf := range configStash {
// look up the real struct type
if st, ok := LoadedCrossExchangeStrategies[id]; ok {
val, err := reUnmarshal(conf, st)
if err != nil {
return err
}
config.CrossExchangeStrategies = append(config.CrossExchangeStrategies, val.(CrossExchangeStrategy))
}
}
}
return nil
}
func NewStrategyFromMap(id string, conf interface{}) (SingleExchangeStrategy, error) {
if st, ok := LoadedExchangeStrategies[id]; ok {
val, err := reUnmarshal(conf, st)
if err != nil {
return nil, err
}
return val.(SingleExchangeStrategy), nil
}
return nil, fmt.Errorf("strategy %s not found", id)
}
func loadExchangeStrategies(config *Config, stash Stash) (err error) {
exchangeStrategiesConf, ok := stash["exchangeStrategies"]
if !ok {
exchangeStrategiesConf, ok = stash["strategies"]
if !ok {
return nil
}
}
if len(LoadedExchangeStrategies) == 0 {
return errors.New("no exchange strategy is registered")
}
configList, ok := exchangeStrategiesConf.([]interface{})
if !ok {
return errors.New("expecting list in exchangeStrategies")
}
for _, entry := range configList {
configStash, ok := entry.(Stash)
if !ok {
return fmt.Errorf("strategy config should be a map, given: %T %+v", entry, entry)
}
var mounts []string
if val, ok := configStash["on"]; ok {
switch tv := val.(type) {
case []string:
mounts = append(mounts, tv...)
case string:
mounts = append(mounts, tv)
case []interface{}:
for _, f := range tv {
s, ok := f.(string)
if !ok {
return fmt.Errorf("%+v (%T) is not a string", f, f)
}
mounts = append(mounts, s)
}
default:
return fmt.Errorf("unexpected mount type: %T value: %+v", val, val)
}
}
for id, conf := range configStash {
// look up the real struct type
if _, ok := LoadedExchangeStrategies[id]; ok {
st, err := NewStrategyFromMap(id, conf)
if err != nil {
return err
}
config.ExchangeStrategies = append(config.ExchangeStrategies, ExchangeStrategyMount{
Mounts: mounts,
Strategy: st,
})
} else if id != "on" && id != "off" {
// Show error when we didn't find the Strategy
return fmt.Errorf("strategy %s in config not found", id)
}
}
}
return nil
}
func reUnmarshal(conf interface{}, tpe interface{}) (interface{}, error) {
// get the type "*Strategy"
rt := reflect.TypeOf(tpe)
// allocate new object from the given type
val := reflect.New(rt)
// now we have &(*Strategy) -> **Strategy
valRef := val.Interface()
plain, err := json.Marshal(conf)
if err != nil {
return nil, err
}
if err := json.Unmarshal(plain, valRef); err != nil {
return nil, errors.Wrapf(err, "json parsing error, given payload: %s", plain)
}
return val.Elem().Interface(), nil
}