From b3da6caddbb2dc52c0441c834ea3f974538a45fc Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 19 May 2022 20:31:25 +0800 Subject: [PATCH] optimizer: fix op builder --- config/optimizer.yaml | 4 +- pkg/backtest/manifests.go | 19 +++++++++ pkg/backtest/report.go | 11 +++++ pkg/cmd/backtest.go | 14 +++--- pkg/optimizer/grid.go | 89 ++++++++++++++++++++++++++++----------- pkg/optimizer/local.go | 44 ++++++++++++------- 6 files changed, 132 insertions(+), 49 deletions(-) diff --git a/config/optimizer.yaml b/config/optimizer.yaml index 2a389cf2e..205ca8ea0 100644 --- a/config/optimizer.yaml +++ b/config/optimizer.yaml @@ -12,10 +12,10 @@ matrix: path: '/exchangeStrategies/0/bollmaker/amount' min: 20.0 max: 100.0 - step: 10.0 + step: 20.0 - type: range path: '/exchangeStrategies/0/bollmaker/spread' min: 0.1% max: 0.2% - step: 0.01% + step: 0.02% diff --git a/pkg/backtest/manifests.go b/pkg/backtest/manifests.go index 932ce02fa..c457e91f5 100644 --- a/pkg/backtest/manifests.go +++ b/pkg/backtest/manifests.go @@ -12,6 +12,25 @@ type ManifestEntry struct { type Manifests map[InstancePropertyIndex]string +func (m *Manifests) UnmarshalJSON(j []byte) error { + var entries []ManifestEntry + if err := json.Unmarshal(j, &entries); err != nil { + return err + } + + mm := make(Manifests) + for _, entry := range entries { + index := InstancePropertyIndex{ + ID: entry.StrategyID, + InstanceID: entry.StrategyInstance, + Property: entry.StrategyProperty, + } + mm[index] = entry.Filename + } + *m = mm + return nil +} + func (m Manifests) MarshalJSON() ([]byte, error) { var arr []ManifestEntry for k, v := range m { diff --git a/pkg/backtest/report.go b/pkg/backtest/report.go index 88f7b98c7..782e6e4d6 100644 --- a/pkg/backtest/report.go +++ b/pkg/backtest/report.go @@ -48,6 +48,17 @@ type SummaryReport struct { Manifests Manifests `json:"manifests,omitempty"` } +func ReadSummaryReport(filename string) (*SummaryReport, error) { + o, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + var report SummaryReport + err = json.Unmarshal(o, &report) + return &report, err +} + // SessionSymbolReport is the report per exchange session // trades are merged, collected and re-calculated type SessionSymbolReport struct { diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 59b1bfdeb..eef9b35b3 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -452,13 +452,6 @@ var BacktestCmd = &cobra.Command{ finalTotalBalances = finalTotalBalances.Add(finalBalances) } - color.Green("BACK-TEST REPORT") - color.Green("===============================================\n") - color.Green("START TIME: %s\n", startTime.Format(time.RFC1123)) - color.Green("END TIME: %s\n", endTime.Format(time.RFC1123)) - color.Green("INITIAL TOTAL BALANCE: %v\n", initTotalBalances) - color.Green("FINAL TOTAL BALANCE: %v\n", finalTotalBalances) - summaryReport := &backtest.SummaryReport{ StartTime: startTime, EndTime: endTime, @@ -526,6 +519,13 @@ var BacktestCmd = &cobra.Command{ } } } else { + color.Green("BACK-TEST REPORT") + color.Green("===============================================\n") + color.Green("START TIME: %s\n", startTime.Format(time.RFC1123)) + color.Green("END TIME: %s\n", endTime.Format(time.RFC1123)) + color.Green("INITIAL TOTAL BALANCE: %v\n", initTotalBalances) + color.Green("FINAL TOTAL BALANCE: %v\n", finalTotalBalances) + for _, symbolReport := range summaryReport.SymbolReports { symbolReport.Print(wantBaseAssetBaseline) } diff --git a/pkg/optimizer/grid.go b/pkg/optimizer/grid.go index 74cc82bb7..dfbe89497 100644 --- a/pkg/optimizer/grid.go +++ b/pkg/optimizer/grid.go @@ -5,17 +5,37 @@ import ( "github.com/evanphx/json-patch/v5" + "github.com/c9s/bbgo/pkg/backtest" "github.com/c9s/bbgo/pkg/fixedpoint" ) +type MetricValueFunc func(summaryReport *backtest.SummaryReport) fixedpoint.Value + +var TotalProfitMetricValueFunc = func(summaryReport *backtest.SummaryReport) fixedpoint.Value { + return summaryReport.TotalProfit +} + +type Metric struct { + Params []interface{} + Value fixedpoint.Value +} + type GridOptimizer struct { Config *Config + + CurrentParams []interface{} + + Metrics []Metric } func (o *GridOptimizer) buildOps() []OpFunc { var ops []OpFunc - for _, selector := range o.Config.Matrix { + + o.CurrentParams = make([]interface{}, len(o.Config.Matrix)) + + for i, selector := range o.Config.Matrix { var path = selector.Path + var ii = i // copy variable because we need to use them in the closure switch selector.Type { case "range": @@ -26,13 +46,12 @@ func (o *GridOptimizer) buildOps() []OpFunc { step = fixedpoint.One } - f := func(configJson []byte, next func(configJson []byte) error) error { - var values []fixedpoint.Value - for val := min; val.Compare(max) < 0; val = val.Add(step) { - values = append(values, val) - } + var values []fixedpoint.Value + for val := min; val.Compare(max) <= 0; val = val.Add(step) { + values = append(values, val) + } - log.Debugf("ranged values: %v", values) + f := func(configJson []byte, next func(configJson []byte) error) error { for _, val := range values { jsonOp := []byte(fmt.Sprintf(`[ {"op": "replace", "path": "%s", "value": %v } ]`, path, val)) patch, err := jsonpatch.DecodePatch(jsonOp) @@ -42,12 +61,14 @@ func (o *GridOptimizer) buildOps() []OpFunc { log.Debugf("json op: %s", jsonOp) - configJson, err := patch.ApplyIndent(configJson, " ") + patchedJson, err := patch.ApplyIndent(configJson, " ") if err != nil { return err } - if err := next(configJson); err != nil { + valCopy := val + o.CurrentParams[ii] = valCopy + if err := next(patchedJson); err != nil { return err } } @@ -59,8 +80,9 @@ func (o *GridOptimizer) buildOps() []OpFunc { case "iterate": values := selector.Values f := func(configJson []byte, next func(configJson []byte) error) error { - log.Debugf("iterate values: %v", values) for _, val := range values { + log.Debugf("%d %s: %v of %v", ii, path, val, values) + jsonOp := []byte(fmt.Sprintf(`[{"op": "replace", "path": "%s", "value": "%s"}]`, path, val)) patch, err := jsonpatch.DecodePatch(jsonOp) if err != nil { @@ -69,12 +91,14 @@ func (o *GridOptimizer) buildOps() []OpFunc { log.Debugf("json op: %s", jsonOp) - configJson, err := patch.ApplyIndent(configJson, " ") + patchedJson, err := patch.ApplyIndent(configJson, " ") if err != nil { return err } - if err := next(configJson); err != nil { + valCopy := val + o.CurrentParams[ii] = valCopy + if err := next(patchedJson); err != nil { return err } } @@ -88,21 +112,38 @@ func (o *GridOptimizer) buildOps() []OpFunc { } func (o *GridOptimizer) Run(executor Executor, configJson []byte) error { - var ops = o.buildOps() - var last = func(configJson []byte, next func(configJson []byte) error) error { - return executor.Execute(configJson) - } - ops = append(ops, last) + o.CurrentParams = make([]interface{}, len(o.Config.Matrix)) - var wrapper = func(configJson []byte) error { return nil } - for i := len(ops) - 1; i > 0; i-- { - next := ops[i] - cur := ops[i-1] + var ops = o.buildOps() + var app = func(configJson []byte, next func(configJson []byte) error) error { + summaryReport, err := executor.Execute(configJson) + if err != nil { + return err + } + + // TODO: Add other metric value function + metricValue := TotalProfitMetricValueFunc(summaryReport) + + o.Metrics = append(o.Metrics, Metric{ + Params: o.CurrentParams, + Value: metricValue, + }) + + log.Infof("current params: %+v => %+v", o.CurrentParams, metricValue) + return nil + } + + log.Debugf("build %d ops", len(ops)) + + var wrapper = func(configJson []byte) error { + return app(configJson, nil) + } + + for i := len(ops) - 1; i >= 0; i-- { + cur := ops[i] inner := wrapper wrapper = func(configJson []byte) error { - return cur(configJson, func(configJson []byte) error { - return next(configJson, inner) - }) + return cur(configJson, inner) } } diff --git a/pkg/optimizer/local.go b/pkg/optimizer/local.go index a408fe163..f9b2979ed 100644 --- a/pkg/optimizer/local.go +++ b/pkg/optimizer/local.go @@ -4,51 +4,63 @@ import ( "encoding/json" "os" "os/exec" + "strings" "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" + + "github.com/c9s/bbgo/pkg/backtest" ) var log = logrus.WithField("component", "optimizer") type Executor interface { - Execute(configJson []byte) error + Execute(configJson []byte) (*backtest.SummaryReport, error) } type LocalProcessExecutor struct { - Bin string - WorkDir string - ConfigDir string - OutputDir string - CombineOutput bool + Bin string + WorkDir string + ConfigDir string + OutputDir string } -func (e *LocalProcessExecutor) Execute(configJson []byte) error { +func (e *LocalProcessExecutor) Execute(configJson []byte) (*backtest.SummaryReport, error) { var o map[string]interface{} if err := json.Unmarshal(configJson, &o); err != nil { - return err + return nil, err } yamlConfig, err := yaml.Marshal(o) if err != nil { - return err + return nil, err } tf, err := os.CreateTemp(e.ConfigDir, "bbgo-*.yaml") if err != nil { - return err + return nil, err } if _, err = tf.Write(yamlConfig); err != nil { - return err + return nil, err } c := exec.Command(e.Bin, "backtest", "--config", tf.Name(), "--output", e.OutputDir, "--subdir") - // c.Output() - if e.CombineOutput { - c.Stdout = os.Stdout - c.Stderr = os.Stderr + output, err := c.Output() + if err != nil { + return nil, err } - return c.Run() + summaryReportFilepath := strings.TrimSpace(string(output)) + _, err = os.Stat(summaryReportFilepath) + if os.IsNotExist(err) { + return nil, err + } + + summaryReport, err := backtest.ReadSummaryReport(summaryReportFilepath) + if err != nil { + return nil, err + } + + return summaryReport, nil }