From a029509b636410f4bdb8005e572cca77003f00f9 Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Wed, 29 Jun 2022 16:17:43 +0800 Subject: [PATCH] optimizer: add progressbar --- go.mod | 6 ++++-- go.sum | 12 ++++++++++++ pkg/optimizer/grid.go | 19 +++++++++++++------ pkg/optimizer/local.go | 12 ++++++++---- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index d7d2f80d6..d1ce9dbf6 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/adshao/go-binance/v2 v2.3.5 github.com/c9s/requestgen v1.3.0 github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b + github.com/cheggaaa/pb/v3 v3.0.8 github.com/codingconcepts/env v0.0.0-20200821220118-a8fbf8d84482 github.com/evanphx/json-patch/v5 v5.6.0 github.com/fatih/camelcase v1.0.0 @@ -55,6 +56,7 @@ require ( ) require ( + github.com/VividCortex/ewma v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect @@ -86,19 +88,19 @@ require ( github.com/magiconair/properties v1.8.4 // indirect github.com/mattn/go-colorable v0.1.9 // indirect github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-runewidth v0.0.12 // indirect github.com/mattn/go-sqlite3 v1.14.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect - github.com/muesli/clusters v0.0.0-20180605185049-a07a36e67d36 // indirect - github.com/muesli/kmeans v0.3.0 // indirect github.com/pelletier/go-toml v1.8.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect diff --git a/go.sum b/go.sum index e1615e021..5aa950f06 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/Masterminds/squirrel v1.5.3 h1:YPpoceAcxuzIljlr5iWpNKaql7hLeG1KLSrhvdHpkZc= github.com/Masterminds/squirrel v1.5.3/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/adshao/go-binance/v2 v2.3.5 h1:WVYZecm0w8l14YoWlnKZj6xxZT2AKMTHpMQSqIX1xxA= github.com/adshao/go-binance/v2 v2.3.5/go.mod h1:8Pg/FGTLyAhq8QXA0IkoReKyRpoxJcK3LVujKDAZV/c= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= @@ -77,6 +79,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA= +github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -122,6 +126,7 @@ github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239/go.mod h1:Gdwt2ce0 github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= @@ -334,6 +339,7 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY= github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -341,6 +347,8 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -434,6 +442,9 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -698,6 +709,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/optimizer/grid.go b/pkg/optimizer/grid.go index c517b2a70..b9aec0f59 100644 --- a/pkg/optimizer/grid.go +++ b/pkg/optimizer/grid.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/cheggaaa/pb/v3" "sort" "github.com/evanphx/json-patch/v5" @@ -172,7 +173,7 @@ func (o *GridOptimizer) Run(executor Executor, configJson []byte) (map[string][] var ops = o.buildOps() - var taskC = make(chan BacktestTask, 100) + var taskC = make(chan BacktestTask, 10000) var app = func(configJson []byte, next func(configJson []byte) error) error { var labels = copyLabels(o.ParamLabels) @@ -200,20 +201,25 @@ func (o *GridOptimizer) Run(executor Executor, configJson []byte) (map[string][] } ctx := context.Background() - resultsC, err := executor.Run(ctx, taskC) + if err := wrapper(configJson); err != nil { + return nil, err + } + + bar := pb.Full.Start(len(taskC)) + bar.SetTemplateString(`{{ string . "log" | green}} | {{counters . }} {{bar . }} {{percent . }} {{etime . }} {{rtime . "ETA %s"}}`) + + resultsC, err := executor.Run(ctx, taskC, bar) if err != nil { return nil, err } - if err := wrapper(configJson); err != nil { - return nil, err - } close(taskC) // this will shut down the executor for result := range resultsC { for metricName, metricFunc := range valueFunctions { var metricValue = metricFunc(result.Report) - log.Infof("params: %+v => %s %+v", result.Params, metricName, metricValue) + bar.Set("log", fmt.Sprintf("params: %+v => %s %+v", result.Params, metricName, metricValue)) + bar.Increment() metrics[metricName] = append(metrics[metricName], Metric{ Params: result.Params, Labels: result.Labels, @@ -221,6 +227,7 @@ func (o *GridOptimizer) Run(executor Executor, configJson []byte) (map[string][] }) } } + bar.Finish() for n := range metrics { sort.Slice(metrics[n], func(i, j int) bool { diff --git a/pkg/optimizer/local.go b/pkg/optimizer/local.go index 393f9703f..c8ab53246 100644 --- a/pkg/optimizer/local.go +++ b/pkg/optimizer/local.go @@ -3,6 +3,8 @@ package optimizer import ( "context" "encoding/json" + "fmt" + "github.com/cheggaaa/pb/v3" "os" "os/exec" "strings" @@ -26,7 +28,7 @@ type BacktestTask struct { type Executor interface { // Execute(configJson []byte) (*backtest.SummaryReport, error) - Run(ctx context.Context, taskC chan BacktestTask) (chan BacktestTask, error) + Run(ctx context.Context, taskC chan BacktestTask, bar *pb.ProgressBar) (chan BacktestTask, error) } type AsyncHandle struct { @@ -73,7 +75,7 @@ func (e *LocalProcessExecutor) readReport(output []byte) (*backtest.SummaryRepor return summaryReport, nil } -func (e *LocalProcessExecutor) Run(ctx context.Context, taskC chan BacktestTask) (chan BacktestTask, error) { +func (e *LocalProcessExecutor) Run(ctx context.Context, taskC chan BacktestTask, bar *pb.ProgressBar) (chan BacktestTask, error) { var maxNumOfProcess = e.Config.MaxNumberOfProcesses var resultsC = make(chan BacktestTask, maxNumOfProcess*2) @@ -89,7 +91,8 @@ func (e *LocalProcessExecutor) Run(ctx context.Context, taskC chan BacktestTask) // fork workers go func(id int, taskC chan BacktestTask) { taskCnt := 0 - log.Infof("starting local worker #%d", id) + bar.Set("log", fmt.Sprintf("starting local worker #%d", id)) + bar.Write() defer wg.Done() for { select { @@ -102,7 +105,8 @@ func (e *LocalProcessExecutor) Run(ctx context.Context, taskC chan BacktestTask) } taskCnt++ - log.Infof("local worker #%d received param task: %v", id, task.Params) + bar.Set("log", fmt.Sprintf("local worker #%d received param task: %v", id, task.Params)) + bar.Write() report, err := e.execute(task.ConfigJson) if err != nil {