From 9d0898ecc66f03de3f415f7138a75fc5f9aa36f3 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Nov 2024 16:41:16 +0800 Subject: [PATCH 1/8] add net profit arg to the position insert method --- pkg/bbgo/environment.go | 4 ++-- pkg/service/position.go | 6 +++++- pkg/service/position_test.go | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index e94e1251f..44f02b9ae 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -587,7 +587,7 @@ func (environ *Environment) RecordPosition(position *types.Position, trade types log.Infof("recordPosition: position = %s, trade = %+v, profit = %+v", position.Base.String(), trade, profit) if profit != nil { - if err := environ.PositionService.Insert(position, trade, profit.Profit); err != nil { + if err := environ.PositionService.Insert(position, trade, profit.Profit, profit.NetProfit); err != nil { log.WithError(err).Errorf("can not insert position record") } @@ -595,7 +595,7 @@ func (environ *Environment) RecordPosition(position *types.Position, trade types log.WithError(err).Errorf("can not insert profit record: %+v", profit) } } else { - if err := environ.PositionService.Insert(position, trade, fixedpoint.Zero); err != nil { + if err := environ.PositionService.Insert(position, trade, fixedpoint.Zero, fixedpoint.Zero); err != nil { log.WithError(err).Errorf("can not insert position record") } } diff --git a/pkg/service/position.go b/pkg/service/position.go index 5fb38eaa7..d09b48bcf 100644 --- a/pkg/service/position.go +++ b/pkg/service/position.go @@ -51,7 +51,11 @@ func (s *PositionService) scanRows(rows *sqlx.Rows) (positions []types.Position, return positions, rows.Err() } -func (s *PositionService) Insert(position *types.Position, trade types.Trade, profit fixedpoint.Value) error { +func (s *PositionService) Insert( + position *types.Position, + trade types.Trade, + profit, netProfit fixedpoint.Value, +) error { _, err := s.DB.NamedExec(` INSERT INTO positions ( strategy, diff --git a/pkg/service/position_test.go b/pkg/service/position_test.go index b45a95fcf..7340f4811 100644 --- a/pkg/service/position_test.go +++ b/pkg/service/position_test.go @@ -34,7 +34,7 @@ func TestPositionService(t *testing.T) { ChangedAt: time.Now(), }, types.Trade{ Time: types.Time(time.Now()), - }, fixedpoint.Zero) + }, fixedpoint.Zero, fixedpoint.Zero) assert.NoError(t, err) }) @@ -54,7 +54,7 @@ func TestPositionService(t *testing.T) { Exchange: types.ExchangeBinance, Side: types.SideTypeSell, Time: types.Time(time.Now()), - }, fixedpoint.NewFromFloat(10.9)) + }, fixedpoint.NewFromFloat(10.9), fixedpoint.Zero) assert.NoError(t, err) }) From ee2b0842f53a362ad8db0aed0c4e8c9738fe30cc Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Nov 2024 16:53:47 +0800 Subject: [PATCH 2/8] migrations: add migration sql files for adding new net_profit column --- .../20241115165059_add_net_profit_column.sql | 15 +++++++++++++++ .../20241115165059_add_net_profit_column.sql | 15 +++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 migrations/mysql/20241115165059_add_net_profit_column.sql create mode 100644 migrations/sqlite3/20241115165059_add_net_profit_column.sql diff --git a/migrations/mysql/20241115165059_add_net_profit_column.sql b/migrations/mysql/20241115165059_add_net_profit_column.sql new file mode 100644 index 000000000..73bdfcf50 --- /dev/null +++ b/migrations/mysql/20241115165059_add_net_profit_column.sql @@ -0,0 +1,15 @@ +-- +up +-- +begin +ALTER TABLE `positions` + ADD COLUMN `net_profit` DECIMAL(16, 8) DEFAULT 0.00000000 NOT NULL +; +-- +end + + +-- +down + +-- +begin +ALTER TABLE `positions` +DROP COLUMN `net_profit` +; +-- +end diff --git a/migrations/sqlite3/20241115165059_add_net_profit_column.sql b/migrations/sqlite3/20241115165059_add_net_profit_column.sql new file mode 100644 index 000000000..70595bb99 --- /dev/null +++ b/migrations/sqlite3/20241115165059_add_net_profit_column.sql @@ -0,0 +1,15 @@ +-- +up +-- +begin +ALTER TABLE `positions` + ADD COLUMN `net_profit` DECIMAL DEFAULT 0.00000000 NOT NULL +; +-- +end + + +-- +down + +-- +begin +ALTER TABLE `positions` +DROP COLUMN `net_profit` +; +-- +end From f2ae512e3fe7190384411f715e50feeb0fbd1e3e Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Nov 2024 16:54:33 +0800 Subject: [PATCH 3/8] compile and update migration package --- ...in_20241115165059_add_net_profit_column.go | 29 +++++++++++++++++++ ...in_20241115165059_add_net_profit_column.go | 29 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 pkg/migrations/mysql/main_20241115165059_add_net_profit_column.go create mode 100644 pkg/migrations/sqlite3/main_20241115165059_add_net_profit_column.go diff --git a/pkg/migrations/mysql/main_20241115165059_add_net_profit_column.go b/pkg/migrations/mysql/main_20241115165059_add_net_profit_column.go new file mode 100644 index 000000000..15307b315 --- /dev/null +++ b/pkg/migrations/mysql/main_20241115165059_add_net_profit_column.go @@ -0,0 +1,29 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper/v2" +) + +func init() { + AddMigration("main", up_main_addNetProfitColumn, down_main_addNetProfitColumn) +} + +func up_main_addNetProfitColumn(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + _, err = tx.ExecContext(ctx, "ALTER TABLE `positions`\n ADD COLUMN `net_profit` DECIMAL(16, 8) DEFAULT 0.00000000 NOT NULL\n;") + if err != nil { + return err + } + return err +} + +func down_main_addNetProfitColumn(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + _, err = tx.ExecContext(ctx, "ALTER TABLE `positions`\nDROP COLUMN `net_profit`\n;") + if err != nil { + return err + } + return err +} diff --git a/pkg/migrations/sqlite3/main_20241115165059_add_net_profit_column.go b/pkg/migrations/sqlite3/main_20241115165059_add_net_profit_column.go new file mode 100644 index 000000000..e397eb71a --- /dev/null +++ b/pkg/migrations/sqlite3/main_20241115165059_add_net_profit_column.go @@ -0,0 +1,29 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper/v2" +) + +func init() { + AddMigration("main", up_main_addNetProfitColumn, down_main_addNetProfitColumn) +} + +func up_main_addNetProfitColumn(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + _, err = tx.ExecContext(ctx, "ALTER TABLE `positions`\n ADD COLUMN `net_profit` DECIMAL DEFAULT 0.00000000 NOT NULL\n;") + if err != nil { + return err + } + return err +} + +func down_main_addNetProfitColumn(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + _, err = tx.ExecContext(ctx, "ALTER TABLE `positions`\nDROP COLUMN `net_profit`\n;") + if err != nil { + return err + } + return err +} From 56fdf0a21f2e36c2b64c66476b47773a50f84b23 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Nov 2024 16:57:00 +0800 Subject: [PATCH 4/8] insert net_profit into the table i really don't know why I forgot this lol --- pkg/service/position.go | 3 +++ pkg/service/position_test.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/service/position.go b/pkg/service/position.go index d09b48bcf..7b379a73a 100644 --- a/pkg/service/position.go +++ b/pkg/service/position.go @@ -67,6 +67,7 @@ func (s *PositionService) Insert( base, quote, profit, + net_profit, trade_id, exchange, side, @@ -81,6 +82,7 @@ func (s *PositionService) Insert( :base, :quote, :profit, + :net_profit, :trade_id, :exchange, :side, @@ -96,6 +98,7 @@ func (s *PositionService) Insert( "base": position.Base, "quote": position.Quote, "profit": profit, + "net_profit": netProfit, "trade_id": trade.ID, "exchange": trade.Exchange, "side": trade.Side, diff --git a/pkg/service/position_test.go b/pkg/service/position_test.go index 7340f4811..d3cb76ef3 100644 --- a/pkg/service/position_test.go +++ b/pkg/service/position_test.go @@ -54,7 +54,7 @@ func TestPositionService(t *testing.T) { Exchange: types.ExchangeBinance, Side: types.SideTypeSell, Time: types.Time(time.Now()), - }, fixedpoint.NewFromFloat(10.9), fixedpoint.Zero) + }, fixedpoint.NewFromFloat(10.9), fixedpoint.NewFromFloat(8.1)) assert.NoError(t, err) }) From 2e39200214f7643c075369eb41b305ff7bcced3f Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Nov 2024 17:25:00 +0800 Subject: [PATCH 5/8] add pprof http server support with a build tag switch --- pkg/cmd/pprof.go | 6 ++++++ pkg/cmd/root.go | 23 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 pkg/cmd/pprof.go diff --git a/pkg/cmd/pprof.go b/pkg/cmd/pprof.go new file mode 100644 index 000000000..e9223b2b4 --- /dev/null +++ b/pkg/cmd/pprof.go @@ -0,0 +1,6 @@ +//go:build pprof +// +build pprof + +package cmd + +import _ "net/http/pprof" diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 2b0ded1df..a696aef56 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -7,6 +7,7 @@ import ( "runtime/pprof" "strings" "time" + _ "time/tzdata" "github.com/heroku/rollrus" "github.com/joho/godotenv" @@ -20,8 +21,6 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/util" - - _ "time/tzdata" ) var cpuProfileFile *os.File @@ -81,6 +80,24 @@ var RootCmd = &cobra.Command{ }() } + enableProfileServer, err := cmd.Flags().GetBool("enable-profile-server") + if err != nil { + return err + } + + if enableProfileServer { + profileServerBind, err := cmd.Flags().GetString("profile-server-bind") + if err != nil { + return err + } + + go func() { + if err := http.ListenAndServe(profileServerBind, nil); err != nil { + log.WithError(err).Errorf("profile server error") + } + }() + } + cpuProfile, err := cmd.Flags().GetString("cpu-profile") if err != nil { return err @@ -195,6 +212,8 @@ func init() { RootCmd.PersistentFlags().String("max-api-secret", "", "max api secret") RootCmd.PersistentFlags().String("cpu-profile", "", "cpu profile") + RootCmd.PersistentFlags().Bool("enable-profile-server", false, "enable profile server binding") + RootCmd.PersistentFlags().String("profile-server-bind", "localhost:6060", "profile server binding") viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) From acf213b9918c08243df01f399fa37821601663b3 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Nov 2024 17:50:16 +0800 Subject: [PATCH 6/8] add profiling doc --- doc/topics/profile.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/topics/profile.md diff --git a/doc/topics/profile.md b/doc/topics/profile.md new file mode 100644 index 000000000..98bb1c1b0 --- /dev/null +++ b/doc/topics/profile.md @@ -0,0 +1,13 @@ +Profiling +=================== + +```shell +dotenv -f .env.local -- go run -tags pprof ./cmd/bbgo run \ + --config ./makemoney-btcusdt.yaml \ + --cpu-profile makemoney.pprof \ + --enable-profile-server +``` + +```shell +go tool pprof http://localhost:6060/debug/pprof/heap +``` From b574c25a56c11c85902e6ffe76efff26b64b4a79 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Nov 2024 18:02:32 +0800 Subject: [PATCH 7/8] migration: set net profit from profit --- migrations/mysql/20241115165059_add_net_profit_column.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/migrations/mysql/20241115165059_add_net_profit_column.sql b/migrations/mysql/20241115165059_add_net_profit_column.sql index 73bdfcf50..702686276 100644 --- a/migrations/mysql/20241115165059_add_net_profit_column.sql +++ b/migrations/mysql/20241115165059_add_net_profit_column.sql @@ -4,7 +4,9 @@ ALTER TABLE `positions` ADD COLUMN `net_profit` DECIMAL(16, 8) DEFAULT 0.00000000 NOT NULL ; -- +end - +-- +begin +UPDATE positions SET net_profit = profit WHERE net_profit = 0.0; +-- +end -- +down From 966a3b3c4582e889d80cb77fbcaba082003a3ce8 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Nov 2024 18:02:50 +0800 Subject: [PATCH 8/8] compile and update migration package --- .../mysql/main_20241115165059_add_net_profit_column.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/migrations/mysql/main_20241115165059_add_net_profit_column.go b/pkg/migrations/mysql/main_20241115165059_add_net_profit_column.go index 15307b315..d17c81c34 100644 --- a/pkg/migrations/mysql/main_20241115165059_add_net_profit_column.go +++ b/pkg/migrations/mysql/main_20241115165059_add_net_profit_column.go @@ -16,6 +16,10 @@ func up_main_addNetProfitColumn(ctx context.Context, tx rockhopper.SQLExecutor) if err != nil { return err } + _, err = tx.ExecContext(ctx, "UPDATE positions SET net_profit = profit WHERE net_profit = 0.0;") + if err != nil { + return err + } return err }