From 7056853ecd49edafb46a80052b38d00b8b9f1278 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 19 May 2022 18:23:12 +0800 Subject: [PATCH] implement grid optimizer and local process executor --- pkg/cmd/optimize.go | 121 ++++++-------------------------------- pkg/optimizer/grid.go | 110 ++++++++++++++++++++++++++++++++++ pkg/optimizer/local.go | 55 +++++++++++++++++ pkg/optimizer/operator.go | 3 + 4 files changed, 187 insertions(+), 102 deletions(-) create mode 100644 pkg/optimizer/grid.go create mode 100644 pkg/optimizer/local.go create mode 100644 pkg/optimizer/operator.go diff --git a/pkg/cmd/optimize.go b/pkg/cmd/optimize.go index 57d15bf81..72a7bef55 100644 --- a/pkg/cmd/optimize.go +++ b/pkg/cmd/optimize.go @@ -3,21 +3,18 @@ package cmd import ( "context" "encoding/json" - "fmt" "io/ioutil" "os" - jsonpatch "github.com/evanphx/json-patch/v5" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v3" - "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/optimizer" ) func init() { optimizeCmd.Flags().String("optimizer-config", "optimizer.yaml", "config file") + optimizeCmd.Flags().String("output", "output", "backtest report output directory") RootCmd.AddCommand(optimizeCmd) } @@ -39,6 +36,11 @@ var optimizeCmd = &cobra.Command{ return err } + outputDirectory, err := cmd.Flags().GetString("output") + if err != nil { + return err + } + yamlBody, err := ioutil.ReadFile(configFile) if err != nil { return err @@ -60,112 +62,27 @@ var optimizeCmd = &cobra.Command{ if err != nil { return err } - log.Info(string(configJson)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() _ = ctx - log.Info(os.Args) - binary := os.Args[0] - _ = binary - - type OpFunc func(configJson []byte, next func(configJson []byte) error) error - var ops []OpFunc - - for _, selector := range optConfig.Matrix { - path := selector.Path - - switch selector.Type { - case "range": - min := selector.Min - max := selector.Max - step := selector.Step - if step.IsZero() { - step = fixedpoint.One - } - - f := func(configJson []byte, next func(configJson []byte) error) error { - var values []fixedpoint.Value - for val := min; val.Compare(max) < 0; val = val.Add(step) { - values = append(values, val) - } - - log.Infof("ranged values: %v", values) - for _, val := range values { - jsonOp := []byte(fmt.Sprintf(`[ {"op": "replace", "path": "%s", "value": %v } ]`, path, val)) - patch, err := jsonpatch.DecodePatch(jsonOp) - if err != nil { - return err - } - - log.Debugf("json op: %s", jsonOp) - - configJson, err := patch.ApplyIndent(configJson, " ") - if err != nil { - return err - } - - if err := next(configJson); err != nil { - return err - } - } - - return nil - } - ops = append(ops, f) - - case "iterate": - values := selector.Values - f := func(configJson []byte, next func(configJson []byte) error) error { - log.Infof("iterate values: %v", values) - for _, val := range values { - jsonOp := []byte(fmt.Sprintf(`[{"op": "replace", "path": "%s", "value": "%s"}]`, path, val)) - patch, err := jsonpatch.DecodePatch(jsonOp) - if err != nil { - return err - } - - log.Debugf("json op: %s", jsonOp) - - configJson, err := patch.ApplyIndent(configJson, " ") - if err != nil { - return err - } - - if err := next(configJson); err != nil { - return err - } - } - - return nil - } - ops = append(ops, f) - } + configDir, err := os.MkdirTemp("", "bbgo-config-*") + if err != nil { + return err } - var last = func(configJson []byte, next func(configJson []byte) error) error { - log.Info("configJson", string(configJson)) - return nil - } - ops = append(ops, last) - - log.Infof("%d ops: %v", len(ops), ops) - - var wrapper = func(configJson []byte) error { return nil } - for i := len(ops) - 1; i > 0; i-- { - next := ops[i] - cur := ops[i-1] - inner := wrapper - a := i - wrapper = func(configJson []byte) error { - log.Infof("wrapper fn #%d", a) - return cur(configJson, func(configJson []byte) error { - return next(configJson, inner) - }) - } + executor := &optimizer.LocalProcessExecutor{ + Bin: os.Args[0], + WorkDir: ".", + ConfigDir: configDir, + OutputDir: outputDirectory, } - return wrapper(configJson) + optz := &optimizer.GridOptimizer{ + Config: optConfig, + } + + return optz.Run(executor, configJson) }, } diff --git a/pkg/optimizer/grid.go b/pkg/optimizer/grid.go new file mode 100644 index 000000000..74cc82bb7 --- /dev/null +++ b/pkg/optimizer/grid.go @@ -0,0 +1,110 @@ +package optimizer + +import ( + "fmt" + + "github.com/evanphx/json-patch/v5" + + "github.com/c9s/bbgo/pkg/fixedpoint" +) + +type GridOptimizer struct { + Config *Config +} + +func (o *GridOptimizer) buildOps() []OpFunc { + var ops []OpFunc + for _, selector := range o.Config.Matrix { + var path = selector.Path + + switch selector.Type { + case "range": + min := selector.Min + max := selector.Max + step := selector.Step + if step.IsZero() { + step = fixedpoint.One + } + + f := func(configJson []byte, next func(configJson []byte) error) error { + var values []fixedpoint.Value + for val := min; val.Compare(max) < 0; val = val.Add(step) { + values = append(values, val) + } + + log.Debugf("ranged values: %v", values) + for _, val := range values { + jsonOp := []byte(fmt.Sprintf(`[ {"op": "replace", "path": "%s", "value": %v } ]`, path, val)) + patch, err := jsonpatch.DecodePatch(jsonOp) + if err != nil { + return err + } + + log.Debugf("json op: %s", jsonOp) + + configJson, err := patch.ApplyIndent(configJson, " ") + if err != nil { + return err + } + + if err := next(configJson); err != nil { + return err + } + } + + return nil + } + ops = append(ops, f) + + case "iterate": + values := selector.Values + f := func(configJson []byte, next func(configJson []byte) error) error { + log.Debugf("iterate values: %v", values) + for _, val := range values { + jsonOp := []byte(fmt.Sprintf(`[{"op": "replace", "path": "%s", "value": "%s"}]`, path, val)) + patch, err := jsonpatch.DecodePatch(jsonOp) + if err != nil { + return err + } + + log.Debugf("json op: %s", jsonOp) + + configJson, err := patch.ApplyIndent(configJson, " ") + if err != nil { + return err + } + + if err := next(configJson); err != nil { + return err + } + } + + return nil + } + ops = append(ops, f) + } + } + return ops +} + +func (o *GridOptimizer) Run(executor Executor, configJson []byte) error { + var ops = o.buildOps() + var last = func(configJson []byte, next func(configJson []byte) error) error { + return executor.Execute(configJson) + } + ops = append(ops, last) + + var wrapper = func(configJson []byte) error { return nil } + for i := len(ops) - 1; i > 0; i-- { + next := ops[i] + cur := ops[i-1] + inner := wrapper + wrapper = func(configJson []byte) error { + return cur(configJson, func(configJson []byte) error { + return next(configJson, inner) + }) + } + } + + return wrapper(configJson) +} diff --git a/pkg/optimizer/local.go b/pkg/optimizer/local.go new file mode 100644 index 000000000..3c1beafb9 --- /dev/null +++ b/pkg/optimizer/local.go @@ -0,0 +1,55 @@ +package optimizer + +import ( + "encoding/json" + "os" + "os/exec" + + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" +) + +var log = logrus.WithField("component", "optimizer") + +type Executor interface { + Execute(configJson []byte) error +} + +type LocalProcessExecutor struct { + Bin string + WorkDir string + ConfigDir string + OutputDir string + CombineOutput bool +} + +func (e *LocalProcessExecutor) Execute(configJson []byte) error { + var o map[string]interface{} + if err := json.Unmarshal(configJson, &o); err != nil { + return err + } + + yamlConfig, err := yaml.Marshal(o) + if err != nil { + return err + } + + tf, err := os.CreateTemp(e.ConfigDir, "bbgo-*.yaml") + if err != nil { + return err + } + + if _, err = tf.Write(yamlConfig); err != nil { + return err + } + + c := exec.Command(e.Bin, "backtest", "--config", tf.Name(), "--output", e.OutputDir, "--subdir") + log.Infof("cmd: %+v", c) + + if e.CombineOutput { + c.Stdout = os.Stdout + c.Stderr = os.Stderr + } + + return c.Run() +} diff --git a/pkg/optimizer/operator.go b/pkg/optimizer/operator.go new file mode 100644 index 000000000..c4ac89cf4 --- /dev/null +++ b/pkg/optimizer/operator.go @@ -0,0 +1,3 @@ +package optimizer + +type OpFunc func(configJson []byte, next func(configJson []byte) error) error