Merge pull request #714 from c9s/improve/sync-symbol-opt

improve: support specifying session in the sync symbol

.github/workflows/go.yml

@@ -63,10 +63,14 @@ jobs:
       run: go build -v ./cmd/bbgo
 
     - name: Test
-      run: go test -race -coverprofile coverage.txt -covermode atomic ./pkg/...
+      run: |
+        go test -race -coverprofile coverage.txt -covermode atomic ./pkg/...
+        sed -i -e '/_requestgen.go/d' coverage.txt
 
     - name: TestDnum
-      run: go test -race -coverprofile coverage_dnum.txt -covermode atomic -tags dnum ./pkg/...
+      run: |
+        go test -race -coverprofile coverage_dnum.txt -covermode atomic -tags dnum ./pkg/...
+        sed -i -e '/_requestgen.go/d' coverage_dnum.txt
 
     - name: Upload Coverage Report
       uses: codecov/codecov-action@v3

config/sync.yaml

@@ -48,6 +48,8 @@ sync:
   - BTCUSDT
   - ETHUSDT
   - DOTUSDT
+  - binance:BNBUSDT
+  - max:USDTTWD
 
   # marginHistory enables the margin history sync
   marginHistory: true

pkg/bbgo/config.go

@@ -208,12 +208,54 @@ func GetNativeBuildTargetConfig() BuildTargetConfig {
 	}
 }
 
+type SyncSymbol struct {
+	Symbol  string `json:"symbol" yaml:"symbol"`
+	Session string `json:"session" yaml:"session"`
+}
+
+func (ss *SyncSymbol) UnmarshalYAML(unmarshal func(a interface{}) error) (err error) {
+	var s string
+	if err = unmarshal(&s); err == nil {
+		aa := strings.SplitN(s, ":", 2)
+		if len(aa) > 1 {
+			ss.Session = aa[0]
+			ss.Symbol = aa[1]
+		} else {
+			ss.Symbol = aa[0]
+		}
+		return nil
+	}
+
+	type localSyncSymbol SyncSymbol
+	var ssNew localSyncSymbol
+	if err = unmarshal(&ssNew); err == nil {
+		*ss = SyncSymbol(ssNew)
+		return nil
+	}
+
+	return err
+}
+
+func categorizeSyncSymbol(slice []SyncSymbol) (map[string][]string, []string) {
+	var rest []string
+	var m = make(map[string][]string)
+	for _, ss := range slice {
+		if len(ss.Session) > 0 {
+			m[ss.Session] = append(m[ss.Session], ss.Symbol)
+		} else {
+			rest = append(rest, ss.Symbol)
+		}
+	}
+	return m, rest
+}
+
 type SyncConfig struct {
 	// Sessions to sync, if ignored, all defined sessions will sync
 	Sessions []string `json:"sessions,omitempty" yaml:"sessions,omitempty"`
 
-	// Symbols is the list of symbol to sync, if ignored, symbols wlll be discovered by your existing crypto balances
-	Symbols []string `json:"symbols,omitempty" yaml:"symbols,omitempty"`
+	// Symbols is the list of session:symbol pair to sync, if ignored, symbols wlll be discovered by your existing crypto balances
+	// Valid formats are: {session}:{symbol},  {symbol} or in YAML object form {symbol: "BTCUSDT", session:"max" }
+	Symbols []SyncSymbol `json:"symbols,omitempty" yaml:"symbols,omitempty"`
 
 	// DepositHistory is for syncing deposit history
 	DepositHistory bool `json:"depositHistory" yaml:"depositHistory"`

pkg/bbgo/config_test.go

@@ -214,5 +214,52 @@ func TestLoadConfig(t *testing.T) {
 			}
 		})
 	}
-
+}
+
+func TestSyncSymbol(t *testing.T) {
+	t.Run("symbol", func(t *testing.T) {
+		var ss []SyncSymbol
+		var err = yaml.Unmarshal([]byte(`- BTCUSDT`), &ss)
+		assert.NoError(t, err)
+		assert.Equal(t, []SyncSymbol{
+			{Symbol: "BTCUSDT"},
+		}, ss)
+	})
+
+	t.Run("session:symbol", func(t *testing.T) {
+		var ss []SyncSymbol
+		var err = yaml.Unmarshal([]byte(`- max:BTCUSDT`), &ss)
+		assert.NoError(t, err)
+		assert.Equal(t, []SyncSymbol{
+			{Session: "max", Symbol: "BTCUSDT"},
+		}, ss)
+	})
+
+	t.Run("object", func(t *testing.T) {
+		var ss []SyncSymbol
+		var err = yaml.Unmarshal([]byte(`- { session: "max", symbol: "BTCUSDT" }`), &ss)
+		assert.NoError(t, err)
+		assert.Equal(t, []SyncSymbol{
+			{Session: "max", Symbol: "BTCUSDT"},
+		}, ss)
+	})
+}
+
+func Test_categorizeSyncSymbol(t *testing.T) {
+	var ss []SyncSymbol
+	var err = yaml.Unmarshal([]byte(`
+- BTCUSDT
+- ETHUSDT
+- max:MAXUSDT
+- max:USDTTWD
+- binance:BNBUSDT
+`), &ss)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, ss)
+
+	sm, rest := categorizeSyncSymbol(ss)
+	assert.NotEmpty(t, rest)
+	assert.NotEmpty(t, sm)
+	assert.Equal(t, []string{"MAXUSDT", "USDTTWD"}, sm["max"])
+	assert.Equal(t, []string{"BNBUSDT"}, sm["binance"])
 }

pkg/bbgo/environment.go

@@ -582,7 +582,6 @@ func (environ *Environment) setSyncing(status SyncStatus) {
 }
 
 func (environ *Environment) syncWithUserConfig(ctx context.Context, userConfig *Config) error {
-	syncSymbols := userConfig.Sync.Symbols
 	sessions := environ.sessions
 	selectedSessions := userConfig.Sync.Sessions
 	if len(selectedSessions) > 0 {
@@ -594,7 +593,13 @@ func (environ *Environment) syncWithUserConfig(ctx context.Context, userConfig *
 		since = userConfig.Sync.Since.Time()
 	}
 
+	syncSymbolMap, restSymbols := categorizeSyncSymbol(userConfig.Sync.Symbols)
 	for _, session := range sessions {
+		syncSymbols := restSymbols
+		if ss, ok := syncSymbolMap[session.Name] ; ok {
+			syncSymbols = append(syncSymbols, ss...)
+		}
+
 		if err := environ.syncSession(ctx, session, syncSymbols...); err != nil {
 			return err
 		}

pkg/cmd/sync.go

@@ -89,7 +89,9 @@ var SyncCmd = &cobra.Command{
 
 		if len(symbol) > 0 {
 			if userConfig.Sync != nil && len(userConfig.Sync.Symbols) > 0 {
-				userConfig.Sync.Symbols = []string{symbol}
+				userConfig.Sync.Symbols = []bbgo.SyncSymbol{
+					{Symbol: symbol},
+				}
 			}
 		}