From 68af2d0ff8803ffe13fd3e7aa2a8c015478d25a2 Mon Sep 17 00:00:00 2001 From: Raphanus Lo Date: Tue, 2 Aug 2022 12:44:42 +0800 Subject: [PATCH] optimizer: rename optimizeex to hoptimize --- config/optimizer-hyperparam-search.yaml | 2 +- pkg/cmd/{optimizeex.go => hoptimize.go} | 77 +++---------- pkg/cmd/optimize.go | 98 +--------------- pkg/optimizer/format.go | 142 ++++++++++++++++++++++++ pkg/optimizer/hpoptimizer.go | 8 +- 5 files changed, 165 insertions(+), 162 deletions(-) rename pkg/cmd/{optimizeex.go => hoptimize.go} (59%) create mode 100644 pkg/optimizer/format.go diff --git a/config/optimizer-hyperparam-search.yaml b/config/optimizer-hyperparam-search.yaml index 0f57225e6..909483c27 100644 --- a/config/optimizer-hyperparam-search.yaml +++ b/config/optimizer-hyperparam-search.yaml @@ -1,6 +1,6 @@ # usage: # -# go run ./cmd/bbgo optimizeex --config bollmaker_ethusdt.yaml --optimizer-config optimizer-hyperparam-search.yaml +# go run ./cmd/bbgo hoptimize --config bollmaker_ethusdt.yaml --optimizer-config optimizer-hyperparam-search.yaml # --- # The search algorithm. Supports the following algorithms: diff --git a/pkg/cmd/optimizeex.go b/pkg/cmd/hoptimize.go similarity index 59% rename from pkg/cmd/optimizeex.go rename to pkg/cmd/hoptimize.go index 1763cd576..ffdc18428 100644 --- a/pkg/cmd/optimizeex.go +++ b/pkg/cmd/hoptimize.go @@ -4,29 +4,27 @@ import ( "context" "encoding/json" "fmt" - "github.com/c9s/bbgo/pkg/data/tsv" "github.com/c9s/bbgo/pkg/optimizer" "github.com/fatih/color" "github.com/spf13/cobra" "gopkg.in/yaml.v3" - "io" "io/ioutil" "os" "time" ) func init() { - optimizeExCmd.Flags().String("optimizer-config", "optimizer.yaml", "config file") - optimizeExCmd.Flags().String("name", "", "assign a name to the study") - optimizeExCmd.Flags().Bool("json-keep-all", false, "keep all results of trials") - optimizeExCmd.Flags().String("output", "output", "backtest report output directory") - optimizeExCmd.Flags().Bool("json", false, "print optimizer metrics in json format") - optimizeExCmd.Flags().Bool("tsv", false, "print optimizer metrics in csv format") - RootCmd.AddCommand(optimizeExCmd) + hoptimizeCmd.Flags().String("optimizer-config", "optimizer.yaml", "config file") + hoptimizeCmd.Flags().String("name", "", "assign an optimization session name") + hoptimizeCmd.Flags().Bool("json-keep-all", false, "keep all results of trials") + hoptimizeCmd.Flags().String("output", "output", "backtest report output directory") + hoptimizeCmd.Flags().Bool("json", false, "print optimizer metrics in json format") + hoptimizeCmd.Flags().Bool("tsv", false, "print optimizer metrics in csv format") + RootCmd.AddCommand(hoptimizeCmd) } -var optimizeExCmd = &cobra.Command{ - Use: "optimizeex", +var hoptimizeCmd = &cobra.Command{ + Use: "hoptimize", Short: "run hyperparameter optimizer (experimental)", // SilenceUsage is an option to silence usage when an error occurs. @@ -43,7 +41,7 @@ var optimizeExCmd = &cobra.Command{ return err } - studyName, err := cmd.Flags().GetString("name") + optSessionName, err := cmd.Flags().GetString("name") if err != nil { return err } @@ -94,10 +92,10 @@ var optimizeExCmd = &cobra.Command{ defer cancel() _ = ctx - if len(studyName) == 0 { - studyName = fmt.Sprintf("bbgo-hpopt-%v", time.Now().UnixMilli()) + if len(optSessionName) == 0 { + optSessionName = fmt.Sprintf("bbgo-hpopt-%v", time.Now().UnixMilli()) } - tempDirNameFormat := fmt.Sprintf("%s-config-*", studyName) + tempDirNameFormat := fmt.Sprintf("%s-config-*", optSessionName) configDir, err := os.MkdirTemp("", tempDirNameFormat) if err != nil { return err @@ -112,8 +110,8 @@ var optimizeExCmd = &cobra.Command{ } optz := &optimizer.HyperparameterOptimizer{ - StudyName: studyName, - Config: optConfig, + SessionName: optSessionName, + Config: optConfig, } if err := executor.Prepare(configJson); err != nil { @@ -137,13 +135,13 @@ var optimizeExCmd = &cobra.Command{ // print report JSON to stdout fmt.Println(string(out)) } else if printTsvFormat { - if err := formatResultsTsv(os.Stdout, report.Parameters, report.Trials); err != nil { + if err := optimizer.FormatResultsTsv(os.Stdout, report.Parameters, report.Trials); err != nil { return err } } else { color.Green("OPTIMIZER REPORT") color.Green("===============================================\n") - color.Green("STUDY NAME: %s\n", report.Name) + color.Green("SESSION NAME: %s\n", report.Name) color.Green("OPTIMIZE OBJECTIVE: %s\n", report.Objective) color.Green("BEST OBJECTIVE VALUE: %s\n", report.Best.Value) color.Green("OPTIMAL PARAMETERS:") @@ -160,44 +158,3 @@ var optimizeExCmd = &cobra.Command{ return nil }, } - -func formatResultsTsv(writer io.WriteCloser, labelPaths map[string]string, results []*optimizer.HyperparameterOptimizeTrialResult) error { - headerLen := len(labelPaths) - headers := make([]string, 0, headerLen) - for label := range labelPaths { - headers = append(headers, label) - } - - rows := make([][]interface{}, len(labelPaths)) - for ri, result := range results { - row := make([]interface{}, headerLen) - for ci, columnKey := range headers { - var ok bool - if row[ci], ok = result.Parameters[columnKey]; !ok { - return fmt.Errorf(`missing parameter "%s" from trial result (%v)`, columnKey, result.Parameters) - } - } - rows[ri] = row - } - - w := tsv.NewWriter(writer) - if err := w.Write(headers); err != nil { - return err - } - - for _, row := range rows { - var cells []string - for _, o := range row { - cell, err := castCellValue(o) - if err != nil { - return err - } - cells = append(cells, cell) - } - - if err := w.Write(cells); err != nil { - return err - } - } - return w.Close() -} diff --git a/pkg/cmd/optimize.go b/pkg/cmd/optimize.go index 15cace8d7..9968cfb2d 100644 --- a/pkg/cmd/optimize.go +++ b/pkg/cmd/optimize.go @@ -4,16 +4,12 @@ import ( "context" "encoding/json" "fmt" - "io" "io/ioutil" "os" - "strconv" "github.com/spf13/cobra" "gopkg.in/yaml.v3" - "github.com/c9s/bbgo/pkg/data/tsv" - "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/optimizer" ) @@ -119,7 +115,7 @@ var optimizeCmd = &cobra.Command{ // print metrics JSON to stdout fmt.Println(string(out)) } else if printTsvFormat { - if err := formatMetricsTsv(metrics, os.Stdout); err != nil { + if err := optimizer.FormatMetricsTsv(os.Stdout, metrics); err != nil { return err } } else { @@ -138,95 +134,3 @@ var optimizeCmd = &cobra.Command{ return nil }, } - -func transformMetricsToRows(metrics map[string][]optimizer.Metric) (headers []string, rows [][]interface{}) { - var metricsKeys []string - for k := range metrics { - metricsKeys = append(metricsKeys, k) - } - - var numEntries int - var paramLabels []string - for _, ms := range metrics { - for _, m := range ms { - paramLabels = m.Labels - break - } - - numEntries = len(ms) - break - } - - headers = append(paramLabels, metricsKeys...) - rows = make([][]interface{}, numEntries) - - var metricsRows = make([][]interface{}, numEntries) - - // build params into the rows - for i, m := range metrics[metricsKeys[0]] { - rows[i] = m.Params - } - - for _, metricKey := range metricsKeys { - for i, ms := range metrics[metricKey] { - if len(metricsRows[i]) == 0 { - metricsRows[i] = make([]interface{}, 0, len(metricsKeys)) - } - metricsRows[i] = append(metricsRows[i], ms.Value) - } - } - - // merge rows - for i := range rows { - rows[i] = append(rows[i], metricsRows[i]...) - } - - return headers, rows -} - -func formatMetricsTsv(metrics map[string][]optimizer.Metric, writer io.WriteCloser) error { - headers, rows := transformMetricsToRows(metrics) - w := tsv.NewWriter(writer) - if err := w.Write(headers); err != nil { - return err - } - - for _, row := range rows { - var cells []string - for _, o := range row { - cell, err := castCellValue(o) - if err != nil { - return err - } - cells = append(cells, cell) - } - - if err := w.Write(cells); err != nil { - return err - } - } - return w.Close() -} - -func castCellValue(a interface{}) (string, error) { - switch tv := a.(type) { - case fixedpoint.Value: - return tv.String(), nil - case float64: - return strconv.FormatFloat(tv, 'f', -1, 64), nil - case int64: - return strconv.FormatInt(tv, 10), nil - case int32: - return strconv.FormatInt(int64(tv), 10), nil - case int: - return strconv.Itoa(tv), nil - case bool: - return strconv.FormatBool(tv), nil - case string: - return tv, nil - case []byte: - return string(tv), nil - default: - return "", fmt.Errorf("unsupported object type: %T value: %v", tv, tv) - } -} diff --git a/pkg/optimizer/format.go b/pkg/optimizer/format.go new file mode 100644 index 000000000..3822c0f30 --- /dev/null +++ b/pkg/optimizer/format.go @@ -0,0 +1,142 @@ +package optimizer + +import ( + "fmt" + "github.com/c9s/bbgo/pkg/data/tsv" + "github.com/c9s/bbgo/pkg/fixedpoint" + "io" + "strconv" +) + +func FormatResultsTsv(writer io.WriteCloser, labelPaths map[string]string, results []*HyperparameterOptimizeTrialResult) error { + headerLen := len(labelPaths) + headers := make([]string, 0, headerLen) + for label := range labelPaths { + headers = append(headers, label) + } + + rows := make([][]interface{}, len(labelPaths)) + for ri, result := range results { + row := make([]interface{}, headerLen) + for ci, columnKey := range headers { + var ok bool + if row[ci], ok = result.Parameters[columnKey]; !ok { + return fmt.Errorf(`missing parameter "%s" from trial result (%v)`, columnKey, result.Parameters) + } + } + rows[ri] = row + } + + w := tsv.NewWriter(writer) + if err := w.Write(headers); err != nil { + return err + } + + for _, row := range rows { + var cells []string + for _, o := range row { + cell, err := castCellValue(o) + if err != nil { + return err + } + cells = append(cells, cell) + } + + if err := w.Write(cells); err != nil { + return err + } + } + return w.Close() +} + +func FormatMetricsTsv(writer io.WriteCloser, metrics map[string][]Metric) error { + headers, rows := transformMetricsToRows(metrics) + w := tsv.NewWriter(writer) + if err := w.Write(headers); err != nil { + return err + } + + for _, row := range rows { + var cells []string + for _, o := range row { + cell, err := castCellValue(o) + if err != nil { + return err + } + cells = append(cells, cell) + } + + if err := w.Write(cells); err != nil { + return err + } + } + return w.Close() +} + +func transformMetricsToRows(metrics map[string][]Metric) (headers []string, rows [][]interface{}) { + var metricsKeys []string + for k := range metrics { + metricsKeys = append(metricsKeys, k) + } + + var numEntries int + var paramLabels []string + for _, ms := range metrics { + for _, m := range ms { + paramLabels = m.Labels + break + } + + numEntries = len(ms) + break + } + + headers = append(paramLabels, metricsKeys...) + rows = make([][]interface{}, numEntries) + + var metricsRows = make([][]interface{}, numEntries) + + // build params into the rows + for i, m := range metrics[metricsKeys[0]] { + rows[i] = m.Params + } + + for _, metricKey := range metricsKeys { + for i, ms := range metrics[metricKey] { + if len(metricsRows[i]) == 0 { + metricsRows[i] = make([]interface{}, 0, len(metricsKeys)) + } + metricsRows[i] = append(metricsRows[i], ms.Value) + } + } + + // merge rows + for i := range rows { + rows[i] = append(rows[i], metricsRows[i]...) + } + + return headers, rows +} + +func castCellValue(a interface{}) (string, error) { + switch tv := a.(type) { + case fixedpoint.Value: + return tv.String(), nil + case float64: + return strconv.FormatFloat(tv, 'f', -1, 64), nil + case int64: + return strconv.FormatInt(tv, 10), nil + case int32: + return strconv.FormatInt(int64(tv), 10), nil + case int: + return strconv.Itoa(tv), nil + case bool: + return strconv.FormatBool(tv), nil + case string: + return tv, nil + case []byte: + return string(tv), nil + default: + return "", fmt.Errorf("unsupported object type: %T value: %v", tv, tv) + } +} diff --git a/pkg/optimizer/hpoptimizer.go b/pkg/optimizer/hpoptimizer.go index ea4494349..7ca777b4c 100644 --- a/pkg/optimizer/hpoptimizer.go +++ b/pkg/optimizer/hpoptimizer.go @@ -75,8 +75,8 @@ func buildHyperparameterOptimizeTrialResults(study *goptuna.Study) []*Hyperparam } type HyperparameterOptimizer struct { - StudyName string - Config *Config + SessionName string + Config *Config // Workaround for goptuna/tpe parameter suggestion. Remove this after fixed. // ref: https://github.com/c-bata/goptuna/issues/236 @@ -112,7 +112,7 @@ func (o *HyperparameterOptimizer) buildStudy(trialFinishChan chan goptuna.Frozen studyOpts = append(studyOpts, goptuna.StudyOptionRelativeSampler(relativeSampler)) } - return goptuna.CreateStudy(o.StudyName, studyOpts...) + return goptuna.CreateStudy(o.SessionName, studyOpts...) } func (o *HyperparameterOptimizer) buildParamDomains() (map[string]string, []paramDomain) { @@ -288,7 +288,7 @@ func (o *HyperparameterOptimizer) Run(executor Executor, configJson []byte) (*Hy bar.Finish() return &HyperparameterOptimizeReport{ - Name: o.StudyName, + Name: o.SessionName, Objective: o.Config.Objective, Parameters: labelPaths, Best: buildBestHyperparameterOptimizeResult(study),