From 51e97284cfc8819dcd06889f389f3cbb4a4fcd6c Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 4 Mar 2022 15:58:59 +0800 Subject: [PATCH 01/32] add profits table migration --- doc/development/migration.md | 15 +++++--- .../mysql/20220304153317_add_profit_table.sql | 38 +++++++++++++++++++ .../20220304153309_add_profit_table.sql | 34 +++++++++++++++++ rockhopper_mysql.yaml | 13 ++++++- 4 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 migrations/mysql/20220304153317_add_profit_table.sql create mode 100644 migrations/sqlite3/20220304153309_add_profit_table.sql diff --git a/doc/development/migration.md b/doc/development/migration.md index 77cf0c11a..47dc911b6 100644 --- a/doc/development/migration.md +++ b/doc/development/migration.md @@ -8,19 +8,22 @@ 2. Create migration files -```sh -rockhopper --config rockhopper_sqlite.yaml create --type sql add_pnl_column -rockhopper --config rockhopper_mysql.yaml create --type sql add_pnl_column -``` -or you can use the util script: +You can use the util script to generate the migration files: ``` bash utils/generate-new-migration.sh add_pnl_column ``` -Be sure to edit both sqlite3 and mysql migration files. ( [Sample](migrations/mysql/20210531234123_add_kline_taker_buy_columns.sql) ) +Or, you can generate the migration files separately: +```sh +rockhopper --config rockhopper_sqlite.yaml create --type sql add_pnl_column +rockhopper --config rockhopper_mysql.yaml create --type sql add_pnl_column +``` + + +Be sure to edit both sqlite3 and mysql migration files. ( [Sample](migrations/mysql/20210531234123_add_kline_taker_buy_columns.sql) ) To test the drivers, you have to update the rockhopper_mysql.yaml file to connect your database, then do: diff --git a/migrations/mysql/20220304153317_add_profit_table.sql b/migrations/mysql/20220304153317_add_profit_table.sql new file mode 100644 index 000000000..cdb60edb4 --- /dev/null +++ b/migrations/mysql/20220304153317_add_profit_table.sql @@ -0,0 +1,38 @@ +-- +up +CREATE TABLE `profits` +( + `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + + `exchange` VARCHAR(24) NOT NULL DEFAULT '', + + `symbol` VARCHAR(8) NOT NULL, + + `trade_id` BIGINT UNSIGNED NOT NULL, + + -- average_cost is the position average cost + `average_cost` DECIMAL(16, 8) UNSIGNED NOT NULL, + + -- profit is the pnl (profit and loss) + `profit` DECIMAL(16, 8) NOT NULL, + + -- price is the price of the trade that makes profit + `price` DECIMAL(16, 8) UNSIGNED NOT NULL, + + -- quantity is the quantity of the trade that makes profit + `quantity` DECIMAL(16, 8) UNSIGNED NOT NULL, + + -- quote_quantity is the quote quantity of the trade that makes profit + `quote_quantity` DECIMAL(16, 8) UNSIGNED NOT NULL, + + -- side is the side of the trade that makes profit + `side` VARCHAR(4) NOT NULL DEFAULT '', + + `traded_at` DATETIME(3) NOT NULL, + + PRIMARY KEY (`gid`), + + UNIQUE KEY `trade_id` (`trade_id`) +); + +-- +down +DROP TABLE IF EXISTS `profits`; diff --git a/migrations/sqlite3/20220304153309_add_profit_table.sql b/migrations/sqlite3/20220304153309_add_profit_table.sql new file mode 100644 index 000000000..73eab6853 --- /dev/null +++ b/migrations/sqlite3/20220304153309_add_profit_table.sql @@ -0,0 +1,34 @@ +-- +up +CREATE TABLE `profits` +( + `gid` INTEGER PRIMARY KEY AUTOINCREMENT, + + `exchange` VARCHAR(24) NOT NULL DEFAULT '', + + `symbol` VARCHAR(8) NOT NULL, + + `trade_id` INTEGER NOT NULL, + + -- average_cost is the position average cost + `average_cost` DECIMAL(16, 8) NOT NULL, + + -- profit is the pnl (profit and loss) + `profit` DECIMAL(16, 8) NOT NULL, + + -- price is the price of the trade that makes profit + `price` DECIMAL(16, 8) NOT NULL, + + -- quantity is the quantity of the trade that makes profit + `quantity` DECIMAL(16, 8) NOT NULL, + + -- quote_quantity is the quote quantity of the trade that makes profit + `quote_quantity` DECIMAL(16, 8) NOT NULL, + + -- side is the side of the trade that makes profit + `side` VARCHAR(4) NOT NULL DEFAULT '', + + `traded_at` DATETIME(3) NOT NULL +); + +-- +down +DROP TABLE IF EXISTS `profits`; diff --git a/rockhopper_mysql.yaml b/rockhopper_mysql.yaml index a0b6e7864..d8b459dbb 100644 --- a/rockhopper_mysql.yaml +++ b/rockhopper_mysql.yaml @@ -1,6 +1,15 @@ +# vim:filetype=yaml: +# you can copy this file to rockhopper_mysql_local.yaml to have your modification --- driver: mysql dialect: mysql -dsn: "root@tcp(localhost:3306)/bbgo_dev?parseTime=true" -# dsn: "root@tcp(localhost:3306)/bbgo_backtest?parseTime=true" + +# unix socket connection to mysql with password +# dsn: "root:123123@unix(/opt/local/var/run/mysql57/mysqld.sock)/bbgo_dev?parseTime=true" + +# tcp connection to mysql with password +# dsn: "root:123123@tcp(localhost:3306)/bbgo_dev?parseTime=true" + +# tcp connection to mysql without password +# dsn: "root@tcp(localhost:3306)/bbgo_dev?parseTime=true" migrationsDir: migrations/mysql From 2bcd3fce453b05dd3ae456203acc5f7e17b1ce89 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 4 Mar 2022 16:04:07 +0800 Subject: [PATCH 02/32] doc: fix doc indentation --- doc/development/migration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/development/migration.md b/doc/development/migration.md index 47dc911b6..50070dc51 100644 --- a/doc/development/migration.md +++ b/doc/development/migration.md @@ -42,9 +42,9 @@ make migrations or ```shell - rockhopper compile --config rockhopper_mysql.yaml --output pkg/migrations/mysql - rockhopper compile --config rockhopper_sqlite.yaml --output pkg/migrations/sqlite3 - git add -v pkg/migrations && git commit -m "compile and update migration package" pkg/migrations || true +rockhopper compile --config rockhopper_mysql.yaml --output pkg/migrations/mysql +rockhopper compile --config rockhopper_sqlite.yaml --output pkg/migrations/sqlite3 +git add -v pkg/migrations && git commit -m "compile and update migration package" pkg/migrations || true ``` From 9e0df77a3643143af041088911653293f1b937ca Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 4 Mar 2022 16:39:48 +0800 Subject: [PATCH 03/32] move profit struct into the types package --- pkg/bbgo/profitstats.go | 341 ---------------------------- pkg/service/profit.go | 59 +++++ pkg/strategy/bollmaker/strategy.go | 6 +- pkg/strategy/grid/strategy.go | 2 +- pkg/strategy/xmaker/state.go | 3 +- pkg/strategy/xmaker/strategy.go | 2 +- pkg/types/profit.go | 343 +++++++++++++++++++++++++++++ 7 files changed, 408 insertions(+), 348 deletions(-) create mode 100644 pkg/service/profit.go create mode 100644 pkg/types/profit.go diff --git a/pkg/bbgo/profitstats.go b/pkg/bbgo/profitstats.go index 0b46307a1..920078f66 100644 --- a/pkg/bbgo/profitstats.go +++ b/pkg/bbgo/profitstats.go @@ -1,343 +1,2 @@ package bbgo -import ( - "fmt" - "github.com/slack-go/slack" - "time" - - "github.com/c9s/bbgo/pkg/fixedpoint" - "github.com/c9s/bbgo/pkg/types" - "github.com/c9s/bbgo/pkg/util" -) - -// Profit struct stores the PnL information -type Profit struct { - Symbol string `json:"symbol"` - - // Profit is the profit of this trade made. negative profit means loss. - Profit fixedpoint.Value `json:"profit" db:"profit"` - - // NetProfit is (profit - trading fee) - NetProfit fixedpoint.Value `json:"netProfit" db:"net_profit"` - AverageCost fixedpoint.Value `json:"averageCost" db:"average_ost"` - - TradeAmount fixedpoint.Value `json:"tradeAmount" db:"trade_amount"` - - // ProfitMargin is a percentage of the profit and the capital amount - ProfitMargin fixedpoint.Value `json:"profitMargin" db:"profit_margin"` - - // NetProfitMargin is a percentage of the net profit and the capital amount - NetProfitMargin fixedpoint.Value `json:"netProfitMargin" db:"net_profit_margin"` - - QuoteCurrency string `json:"quoteCurrency" db:"quote_currency"` - BaseCurrency string `json:"baseCurrency" db:"base_currency"` - - // FeeInUSD is the summed fee of this profit, - // you will need to convert the trade fee into USD since the fee currencies can be different. - FeeInUSD fixedpoint.Value `json:"feeInUSD" db:"fee_in_usd"` - Time time.Time `json:"time" db:"time"` - Strategy string `json:"strategy" db:"strategy"` - StrategyInstanceID string `json:"strategyInstanceID" db:"strategy_instance_id"` -} - -func (p *Profit) SlackAttachment() slack.Attachment { - var color = pnlColor(p.Profit) - var title = fmt.Sprintf("%s PnL ", p.Symbol) - title += pnlEmojiMargin(p.Profit, p.ProfitMargin, defaultPnlLevelResolution) + " " - title += pnlSignString(p.Profit) + " " + p.QuoteCurrency - - var fields []slack.AttachmentField - - if !p.NetProfit.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Net Profit", - Value: pnlSignString(p.NetProfit) + " " + p.QuoteCurrency, - Short: true, - }) - } - - if !p.ProfitMargin.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Profit Margin", - Value: p.ProfitMargin.Percentage(), - Short: true, - }) - } - - if !p.NetProfitMargin.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Net Profit Margin", - Value: p.NetProfitMargin.Percentage(), - Short: true, - }) - } - - if !p.TradeAmount.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Trade Amount", - Value: p.TradeAmount.String() + " " + p.QuoteCurrency, - Short: true, - }) - } - - if !p.FeeInUSD.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Fee In USD", - Value: p.FeeInUSD.String() + " USD", - Short: true, - }) - } - - if len(p.Strategy) != 0 { - fields = append(fields, slack.AttachmentField{ - Title: "Strategy", - Value: p.Strategy, - Short: true, - }) - } - - return slack.Attachment{ - Color: color, - Title: title, - Fields: fields, - // Footer: "", - } -} - -func (p *Profit) PlainText() string { - var emoji string - if !p.ProfitMargin.IsZero() { - emoji = pnlEmojiMargin(p.Profit, p.ProfitMargin, defaultPnlLevelResolution) - } else { - emoji = pnlEmojiSimple(p.Profit) - } - - return fmt.Sprintf("%s trade profit %s %s %s (%s), net profit =~ %s %s (%s)", - p.Symbol, - emoji, - p.Profit.String(), p.QuoteCurrency, - p.ProfitMargin.Percentage(), - p.NetProfit.String(), p.QuoteCurrency, - p.NetProfitMargin.Percentage(), - ) -} - -var lossEmoji = "🔥" -var profitEmoji = "💰" -var defaultPnlLevelResolution = fixedpoint.NewFromFloat(0.001) - -func pnlColor(pnl fixedpoint.Value) string { - if pnl.Sign() > 0 { - return types.GreenColor - } - return types.RedColor -} - -func pnlSignString(pnl fixedpoint.Value) string { - if pnl.Sign() > 0 { - return "+" + pnl.String() - } - return pnl.String() -} - -func pnlEmojiSimple(pnl fixedpoint.Value) string { - if pnl.Sign() < 0 { - return lossEmoji - } - - if pnl.IsZero() { - return "" - } - - return profitEmoji -} - -func pnlEmojiMargin(pnl, margin, resolution fixedpoint.Value) (out string) { - if margin.IsZero() { - return pnlEmojiSimple(pnl) - } - - if pnl.Sign() < 0 { - out = lossEmoji - level := (margin.Neg()).Div(resolution).Int() - for i := 1; i < level; i++ { - out += lossEmoji - } - return out - } - - if pnl.IsZero() { - return out - } - - out = profitEmoji - level := margin.Div(resolution).Int() - for i := 1; i < level; i++ { - out += profitEmoji - } - return out -} - -type ProfitStats struct { - Symbol string `json:"symbol"` - QuoteCurrency string `json:"quoteCurrency"` - BaseCurrency string `json:"baseCurrency"` - - AccumulatedPnL fixedpoint.Value `json:"accumulatedPnL,omitempty"` - AccumulatedNetProfit fixedpoint.Value `json:"accumulatedNetProfit,omitempty"` - AccumulatedProfit fixedpoint.Value `json:"accumulatedProfit,omitempty"` - AccumulatedLoss fixedpoint.Value `json:"accumulatedLoss,omitempty"` - AccumulatedVolume fixedpoint.Value `json:"accumulatedVolume,omitempty"` - AccumulatedSince int64 `json:"accumulatedSince,omitempty"` - - TodayPnL fixedpoint.Value `json:"todayPnL,omitempty"` - TodayNetProfit fixedpoint.Value `json:"todayNetProfit,omitempty"` - TodayProfit fixedpoint.Value `json:"todayProfit,omitempty"` - TodayLoss fixedpoint.Value `json:"todayLoss,omitempty"` - TodaySince int64 `json:"todaySince,omitempty"` -} - -func (s *ProfitStats) Init(market types.Market) { - s.Symbol = market.Symbol - s.BaseCurrency = market.BaseCurrency - s.QuoteCurrency = market.QuoteCurrency - if s.AccumulatedSince == 0 { - s.AccumulatedSince = time.Now().Unix() - } -} - -func (s *ProfitStats) AddProfit(profit Profit) { - s.AccumulatedPnL = s.AccumulatedPnL.Add(profit.Profit) - s.AccumulatedNetProfit = s.AccumulatedNetProfit.Add(profit.NetProfit) - s.TodayPnL = s.TodayPnL.Add(profit.Profit) - s.TodayNetProfit = s.TodayNetProfit.Add(profit.NetProfit) - - if profit.Profit.Sign() < 0 { - s.AccumulatedLoss = s.AccumulatedLoss.Add(profit.Profit) - s.TodayLoss = s.TodayLoss.Add(profit.Profit) - } else if profit.Profit.Sign() > 0 { - s.AccumulatedProfit = s.AccumulatedLoss.Add(profit.Profit) - s.TodayProfit = s.TodayProfit.Add(profit.Profit) - } -} - -func (s *ProfitStats) AddTrade(trade types.Trade) { - if s.IsOver24Hours() { - s.ResetToday() - } - - s.AccumulatedVolume = s.AccumulatedVolume.Add(trade.Quantity) -} - -func (s *ProfitStats) IsOver24Hours() bool { - return time.Since(time.Unix(s.TodaySince, 0)) > 24*time.Hour -} - -func (s *ProfitStats) ResetToday() { - s.TodayPnL = fixedpoint.Zero - s.TodayNetProfit = fixedpoint.Zero - s.TodayProfit = fixedpoint.Zero - s.TodayLoss = fixedpoint.Zero - - var beginningOfTheDay = util.BeginningOfTheDay(time.Now().Local()) - s.TodaySince = beginningOfTheDay.Unix() -} - -func (s *ProfitStats) PlainText() string { - since := time.Unix(s.AccumulatedSince, 0).Local() - return fmt.Sprintf("%s Profit Today\n"+ - "Profit %s %s\n"+ - "Net profit %s %s\n"+ - "Trade Loss %s %s\n"+ - "Summary:\n"+ - "Accumulated Profit %s %s\n"+ - "Accumulated Net Profit %s %s\n"+ - "Accumulated Trade Loss %s %s\n"+ - "Since %s", - s.Symbol, - s.TodayPnL.String(), s.QuoteCurrency, - s.TodayNetProfit.String(), s.QuoteCurrency, - s.TodayLoss.String(), s.QuoteCurrency, - s.AccumulatedPnL.String(), s.QuoteCurrency, - s.AccumulatedNetProfit.String(), s.QuoteCurrency, - s.AccumulatedLoss.String(), s.QuoteCurrency, - since.Format(time.RFC822), - ) -} - -func (s *ProfitStats) SlackAttachment() slack.Attachment { - var color = pnlColor(s.AccumulatedPnL) - var title = fmt.Sprintf("%s Accumulated PnL %s %s", s.Symbol, pnlSignString(s.AccumulatedPnL), s.QuoteCurrency) - - since := time.Unix(s.AccumulatedSince, 0).Local() - title += " Since " + since.Format(time.RFC822) - - var fields []slack.AttachmentField - - if !s.TodayPnL.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "P&L Today", - Value: pnlSignString(s.TodayPnL) + " " + s.QuoteCurrency, - Short: true, - }) - } - - if !s.TodayProfit.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Profit Today", - Value: pnlSignString(s.TodayProfit) + " " + s.QuoteCurrency, - Short: true, - }) - } - - if !s.TodayNetProfit.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Net Profit Today", - Value: pnlSignString(s.TodayNetProfit) + " " + s.QuoteCurrency, - Short: true, - }) - } - - if !s.TodayLoss.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Loss Today", - Value: pnlSignString(s.TodayLoss) + " " + s.QuoteCurrency, - Short: true, - }) - } - - if !s.AccumulatedPnL.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Accumulated P&L", - Value: pnlSignString(s.AccumulatedPnL) + " " + s.QuoteCurrency, - }) - } - - if !s.AccumulatedProfit.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Accumulated Profit", - Value: pnlSignString(s.AccumulatedProfit) + " " + s.QuoteCurrency, - }) - } - - if !s.AccumulatedNetProfit.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Accumulated Net Profit", - Value: pnlSignString(s.AccumulatedNetProfit) + " " + s.QuoteCurrency, - }) - } - - if !s.AccumulatedLoss.IsZero() { - fields = append(fields, slack.AttachmentField{ - Title: "Accumulated Loss", - Value: pnlSignString(s.AccumulatedLoss) + " " + s.QuoteCurrency, - }) - } - - return slack.Attachment{ - Color: color, - Title: title, - Fields: fields, - // Footer: "", - } -} diff --git a/pkg/service/profit.go b/pkg/service/profit.go new file mode 100644 index 000000000..7920d3aca --- /dev/null +++ b/pkg/service/profit.go @@ -0,0 +1,59 @@ +package service + +import ( + "context" + + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + + "github.com/c9s/bbgo/pkg/types" +) + +type ProfitService struct { + DB *sqlx.DB +} + +func NewProfitService(db *sqlx.DB) *ProfitService { + return &ProfitService{db} +} + +func (s *ProfitService) Load(ctx context.Context, id int64) (*types.Trade, error) { + var trade types.Trade + + rows, err := s.DB.NamedQuery("SELECT * FROM trades WHERE id = :id", map[string]interface{}{ + "id": id, + }) + if err != nil { + return nil, err + } + + defer rows.Close() + + if rows.Next() { + err = rows.StructScan(&trade) + return &trade, err + } + + return nil, errors.Wrapf(ErrTradeNotFound, "trade id:%d not found", id) +} + +func (s *ProfitService) scanRows(rows *sqlx.Rows) (trades []types.Trade, err error) { + for rows.Next() { + var trade types.Trade + if err := rows.StructScan(&trade); err != nil { + return trades, err + } + + trades = append(trades, trade) + } + + return trades, rows.Err() +} + +func (s *ProfitService) Insert(trade types.Trade) error { + _, err := s.DB.NamedExec(` + INSERT INTO profits (id, exchange, symbol, trade_id, average_cost, profit, price, quantity, quote_quantity, side, traded_at, is_margin, is_futures, is_isolated) + VALUES (:id, :exchange, :order_id, :symbol, :price, :quantity, :quote_quantity, :side, :is_buyer, :is_maker, :fee, :fee_currency, :traded_at, :is_margin, :is_futures, :is_isolated)`, + trade) + return err +} diff --git a/pkg/strategy/bollmaker/strategy.go b/pkg/strategy/bollmaker/strategy.go index cbf313d94..bdf4f9a1c 100644 --- a/pkg/strategy/bollmaker/strategy.go +++ b/pkg/strategy/bollmaker/strategy.go @@ -38,8 +38,8 @@ func init() { } type State struct { - Position *types.Position `json:"position,omitempty"` - ProfitStats bbgo.ProfitStats `json:"profitStats,omitempty"` + Position *types.Position `json:"position,omitempty"` + ProfitStats types.ProfitStats `json:"profitStats,omitempty"` } type BollingerSetting struct { @@ -571,7 +571,7 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se s.tradeCollector = bbgo.NewTradeCollector(s.Symbol, s.state.Position, s.orderStore) s.tradeCollector.OnProfit(func(trade types.Trade, profit fixedpoint.Value, netProfit fixedpoint.Value) { log.Infof("generated profit: %v", profit) - p := bbgo.Profit{ + p := types.Profit{ Symbol: s.Symbol, Profit: profit, NetProfit: netProfit, diff --git a/pkg/strategy/grid/strategy.go b/pkg/strategy/grid/strategy.go index 572e0833a..f03d2df7b 100644 --- a/pkg/strategy/grid/strategy.go +++ b/pkg/strategy/grid/strategy.go @@ -41,7 +41,7 @@ type State struct { // [source Order ID] -> arbitrage order ArbitrageOrders map[uint64]types.Order `json:"arbitrageOrders"` - ProfitStats bbgo.ProfitStats `json:"profitStats,omitempty"` + ProfitStats types.ProfitStats `json:"profitStats,omitempty"` } type Strategy struct { diff --git a/pkg/strategy/xmaker/state.go b/pkg/strategy/xmaker/state.go index f6a57c285..ea82bc96f 100644 --- a/pkg/strategy/xmaker/state.go +++ b/pkg/strategy/xmaker/state.go @@ -3,7 +3,6 @@ package xmaker import ( "sync" - "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -15,7 +14,7 @@ type State struct { } type ProfitStats struct { - bbgo.ProfitStats + types.ProfitStats lock sync.Mutex MakerExchange types.ExchangeName `json:"makerExchange"` diff --git a/pkg/strategy/xmaker/strategy.go b/pkg/strategy/xmaker/strategy.go index bbfc162b8..cf85e0d9d 100644 --- a/pkg/strategy/xmaker/strategy.go +++ b/pkg/strategy/xmaker/strategy.go @@ -763,7 +763,7 @@ func (s *Strategy) CrossRun(ctx context.Context, orderExecutionRouter bbgo.Order } }) s.tradeCollector.OnProfit(func(trade types.Trade, profit fixedpoint.Value, netProfit fixedpoint.Value) { - p := bbgo.Profit{ + p := types.Profit{ Symbol: s.Symbol, Profit: profit, NetProfit: netProfit, diff --git a/pkg/types/profit.go b/pkg/types/profit.go new file mode 100644 index 000000000..90d06134d --- /dev/null +++ b/pkg/types/profit.go @@ -0,0 +1,343 @@ +package types + +import ( + "fmt" + "time" + + "github.com/slack-go/slack" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/util" +) + +// Profit struct stores the PnL information +type Profit struct { + Symbol string `json:"symbol"` + + // Profit is the profit of this trade made. negative profit means loss. + Profit fixedpoint.Value `json:"profit" db:"profit"` + + // NetProfit is (profit - trading fee) + NetProfit fixedpoint.Value `json:"netProfit" db:"net_profit"` + AverageCost fixedpoint.Value `json:"averageCost" db:"average_ost"` + + TradeAmount fixedpoint.Value `json:"tradeAmount" db:"trade_amount"` + + // ProfitMargin is a percentage of the profit and the capital amount + ProfitMargin fixedpoint.Value `json:"profitMargin" db:"profit_margin"` + + // NetProfitMargin is a percentage of the net profit and the capital amount + NetProfitMargin fixedpoint.Value `json:"netProfitMargin" db:"net_profit_margin"` + + QuoteCurrency string `json:"quoteCurrency" db:"quote_currency"` + BaseCurrency string `json:"baseCurrency" db:"base_currency"` + + // FeeInUSD is the summed fee of this profit, + // you will need to convert the trade fee into USD since the fee currencies can be different. + FeeInUSD fixedpoint.Value `json:"feeInUSD" db:"fee_in_usd"` + Time time.Time `json:"time" db:"time"` + Strategy string `json:"strategy" db:"strategy"` + StrategyInstanceID string `json:"strategyInstanceID" db:"strategy_instance_id"` +} + +func (p *Profit) SlackAttachment() slack.Attachment { + var color = pnlColor(p.Profit) + var title = fmt.Sprintf("%s PnL ", p.Symbol) + title += pnlEmojiMargin(p.Profit, p.ProfitMargin, defaultPnlLevelResolution) + " " + title += pnlSignString(p.Profit) + " " + p.QuoteCurrency + + var fields []slack.AttachmentField + + if !p.NetProfit.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Net Profit", + Value: pnlSignString(p.NetProfit) + " " + p.QuoteCurrency, + Short: true, + }) + } + + if !p.ProfitMargin.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Profit Margin", + Value: p.ProfitMargin.Percentage(), + Short: true, + }) + } + + if !p.NetProfitMargin.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Net Profit Margin", + Value: p.NetProfitMargin.Percentage(), + Short: true, + }) + } + + if !p.TradeAmount.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Trade Amount", + Value: p.TradeAmount.String() + " " + p.QuoteCurrency, + Short: true, + }) + } + + if !p.FeeInUSD.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Fee In USD", + Value: p.FeeInUSD.String() + " USD", + Short: true, + }) + } + + if len(p.Strategy) != 0 { + fields = append(fields, slack.AttachmentField{ + Title: "Strategy", + Value: p.Strategy, + Short: true, + }) + } + + return slack.Attachment{ + Color: color, + Title: title, + Fields: fields, + // Footer: "", + } +} + +func (p *Profit) PlainText() string { + var emoji string + if !p.ProfitMargin.IsZero() { + emoji = pnlEmojiMargin(p.Profit, p.ProfitMargin, defaultPnlLevelResolution) + } else { + emoji = pnlEmojiSimple(p.Profit) + } + + return fmt.Sprintf("%s trade profit %s %s %s (%s), net profit =~ %s %s (%s)", + p.Symbol, + emoji, + p.Profit.String(), p.QuoteCurrency, + p.ProfitMargin.Percentage(), + p.NetProfit.String(), p.QuoteCurrency, + p.NetProfitMargin.Percentage(), + ) +} + +var lossEmoji = "🔥" +var profitEmoji = "💰" +var defaultPnlLevelResolution = fixedpoint.NewFromFloat(0.001) + +func pnlColor(pnl fixedpoint.Value) string { + if pnl.Sign() > 0 { + return GreenColor + } + return RedColor +} + +func pnlSignString(pnl fixedpoint.Value) string { + if pnl.Sign() > 0 { + return "+" + pnl.String() + } + return pnl.String() +} + +func pnlEmojiSimple(pnl fixedpoint.Value) string { + if pnl.Sign() < 0 { + return lossEmoji + } + + if pnl.IsZero() { + return "" + } + + return profitEmoji +} + +func pnlEmojiMargin(pnl, margin, resolution fixedpoint.Value) (out string) { + if margin.IsZero() { + return pnlEmojiSimple(pnl) + } + + if pnl.Sign() < 0 { + out = lossEmoji + level := (margin.Neg()).Div(resolution).Int() + for i := 1; i < level; i++ { + out += lossEmoji + } + return out + } + + if pnl.IsZero() { + return out + } + + out = profitEmoji + level := margin.Div(resolution).Int() + for i := 1; i < level; i++ { + out += profitEmoji + } + return out +} + +type ProfitStats struct { + Symbol string `json:"symbol"` + QuoteCurrency string `json:"quoteCurrency"` + BaseCurrency string `json:"baseCurrency"` + + AccumulatedPnL fixedpoint.Value `json:"accumulatedPnL,omitempty"` + AccumulatedNetProfit fixedpoint.Value `json:"accumulatedNetProfit,omitempty"` + AccumulatedProfit fixedpoint.Value `json:"accumulatedProfit,omitempty"` + AccumulatedLoss fixedpoint.Value `json:"accumulatedLoss,omitempty"` + AccumulatedVolume fixedpoint.Value `json:"accumulatedVolume,omitempty"` + AccumulatedSince int64 `json:"accumulatedSince,omitempty"` + + TodayPnL fixedpoint.Value `json:"todayPnL,omitempty"` + TodayNetProfit fixedpoint.Value `json:"todayNetProfit,omitempty"` + TodayProfit fixedpoint.Value `json:"todayProfit,omitempty"` + TodayLoss fixedpoint.Value `json:"todayLoss,omitempty"` + TodaySince int64 `json:"todaySince,omitempty"` +} + +func (s *ProfitStats) Init(market Market) { + s.Symbol = market.Symbol + s.BaseCurrency = market.BaseCurrency + s.QuoteCurrency = market.QuoteCurrency + if s.AccumulatedSince == 0 { + s.AccumulatedSince = time.Now().Unix() + } +} + +func (s *ProfitStats) AddProfit(profit Profit) { + s.AccumulatedPnL = s.AccumulatedPnL.Add(profit.Profit) + s.AccumulatedNetProfit = s.AccumulatedNetProfit.Add(profit.NetProfit) + s.TodayPnL = s.TodayPnL.Add(profit.Profit) + s.TodayNetProfit = s.TodayNetProfit.Add(profit.NetProfit) + + if profit.Profit.Sign() < 0 { + s.AccumulatedLoss = s.AccumulatedLoss.Add(profit.Profit) + s.TodayLoss = s.TodayLoss.Add(profit.Profit) + } else if profit.Profit.Sign() > 0 { + s.AccumulatedProfit = s.AccumulatedLoss.Add(profit.Profit) + s.TodayProfit = s.TodayProfit.Add(profit.Profit) + } +} + +func (s *ProfitStats) AddTrade(trade Trade) { + if s.IsOver24Hours() { + s.ResetToday() + } + + s.AccumulatedVolume = s.AccumulatedVolume.Add(trade.Quantity) +} + +func (s *ProfitStats) IsOver24Hours() bool { + return time.Since(time.Unix(s.TodaySince, 0)) > 24*time.Hour +} + +func (s *ProfitStats) ResetToday() { + s.TodayPnL = fixedpoint.Zero + s.TodayNetProfit = fixedpoint.Zero + s.TodayProfit = fixedpoint.Zero + s.TodayLoss = fixedpoint.Zero + + var beginningOfTheDay = util.BeginningOfTheDay(time.Now().Local()) + s.TodaySince = beginningOfTheDay.Unix() +} + +func (s *ProfitStats) PlainText() string { + since := time.Unix(s.AccumulatedSince, 0).Local() + return fmt.Sprintf("%s Profit Today\n"+ + "Profit %s %s\n"+ + "Net profit %s %s\n"+ + "Trade Loss %s %s\n"+ + "Summary:\n"+ + "Accumulated Profit %s %s\n"+ + "Accumulated Net Profit %s %s\n"+ + "Accumulated Trade Loss %s %s\n"+ + "Since %s", + s.Symbol, + s.TodayPnL.String(), s.QuoteCurrency, + s.TodayNetProfit.String(), s.QuoteCurrency, + s.TodayLoss.String(), s.QuoteCurrency, + s.AccumulatedPnL.String(), s.QuoteCurrency, + s.AccumulatedNetProfit.String(), s.QuoteCurrency, + s.AccumulatedLoss.String(), s.QuoteCurrency, + since.Format(time.RFC822), + ) +} + +func (s *ProfitStats) SlackAttachment() slack.Attachment { + var color = pnlColor(s.AccumulatedPnL) + var title = fmt.Sprintf("%s Accumulated PnL %s %s", s.Symbol, pnlSignString(s.AccumulatedPnL), s.QuoteCurrency) + + since := time.Unix(s.AccumulatedSince, 0).Local() + title += " Since " + since.Format(time.RFC822) + + var fields []slack.AttachmentField + + if !s.TodayPnL.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "P&L Today", + Value: pnlSignString(s.TodayPnL) + " " + s.QuoteCurrency, + Short: true, + }) + } + + if !s.TodayProfit.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Profit Today", + Value: pnlSignString(s.TodayProfit) + " " + s.QuoteCurrency, + Short: true, + }) + } + + if !s.TodayNetProfit.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Net Profit Today", + Value: pnlSignString(s.TodayNetProfit) + " " + s.QuoteCurrency, + Short: true, + }) + } + + if !s.TodayLoss.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Loss Today", + Value: pnlSignString(s.TodayLoss) + " " + s.QuoteCurrency, + Short: true, + }) + } + + if !s.AccumulatedPnL.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Accumulated P&L", + Value: pnlSignString(s.AccumulatedPnL) + " " + s.QuoteCurrency, + }) + } + + if !s.AccumulatedProfit.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Accumulated Profit", + Value: pnlSignString(s.AccumulatedProfit) + " " + s.QuoteCurrency, + }) + } + + if !s.AccumulatedNetProfit.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Accumulated Net Profit", + Value: pnlSignString(s.AccumulatedNetProfit) + " " + s.QuoteCurrency, + }) + } + + if !s.AccumulatedLoss.IsZero() { + fields = append(fields, slack.AttachmentField{ + Title: "Accumulated Loss", + Value: pnlSignString(s.AccumulatedLoss) + " " + s.QuoteCurrency, + }) + } + + return slack.Attachment{ + Color: color, + Title: title, + Fields: fields, + // Footer: "", + } +} From bb52fcb48f68b127704b8ed427fa170f86eeec3b Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 4 Mar 2022 16:40:01 +0800 Subject: [PATCH 04/32] update profit table columns --- .../mysql/20220304153317_add_profit_table.sql | 60 +++++++++++++------ .../20220304153309_add_profit_table.sql | 59 +++++++++++++----- 2 files changed, 86 insertions(+), 33 deletions(-) diff --git a/migrations/mysql/20220304153317_add_profit_table.sql b/migrations/mysql/20220304153317_add_profit_table.sql index cdb60edb4..5db882b66 100644 --- a/migrations/mysql/20220304153317_add_profit_table.sql +++ b/migrations/mysql/20220304153317_add_profit_table.sql @@ -1,36 +1,62 @@ -- +up CREATE TABLE `profits` ( - `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, - `exchange` VARCHAR(24) NOT NULL DEFAULT '', - - `symbol` VARCHAR(8) NOT NULL, - - `trade_id` BIGINT UNSIGNED NOT NULL, + `symbol` VARCHAR(8) NOT NULL, -- average_cost is the position average cost - `average_cost` DECIMAL(16, 8) UNSIGNED NOT NULL, + `average_cost` DECIMAL(16, 8) UNSIGNED NOT NULL, -- profit is the pnl (profit and loss) - `profit` DECIMAL(16, 8) NOT NULL, + `profit` DECIMAL(16, 8) NOT NULL, - -- price is the price of the trade that makes profit - `price` DECIMAL(16, 8) UNSIGNED NOT NULL, + -- net_profit is the pnl (profit and loss) + `net_profit` DECIMAL(16, 8) NOT NULL, - -- quantity is the quantity of the trade that makes profit - `quantity` DECIMAL(16, 8) UNSIGNED NOT NULL, + -- profit_margin is the pnl (profit and loss) + `profit_margin` DECIMAL(16, 8) NOT NULL, - -- quote_quantity is the quote quantity of the trade that makes profit - `quote_quantity` DECIMAL(16, 8) UNSIGNED NOT NULL, + -- net_profit_margin is the pnl (profit and loss) + `net_profit_margin` DECIMAL(16, 8) NOT NULL, + + `quote_currency` VARCHAR(10) NOT NULL, + + `base_currency` VARCHAR(10) NOT NULL, + + -- ------------------------------------------------------- + -- embedded trade data -- + -- ------------------------------------------------------- + `exchange` VARCHAR(24) NOT NULL DEFAULT '', + + `is_futures` BOOLEAN NOT NULL DEFAULT FALSE, + + `is_margin` BOOLEAN NOT NULL DEFAULT FALSE, + + `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE, + + `trade_id` BIGINT UNSIGNED NOT NULL, -- side is the side of the trade that makes profit - `side` VARCHAR(4) NOT NULL DEFAULT '', + `side` VARCHAR(4) NOT NULL DEFAULT '', - `traded_at` DATETIME(3) NOT NULL, + -- price is the price of the trade that makes profit + `price` DECIMAL(16, 8) UNSIGNED NOT NULL, + + -- quantity is the quantity of the trade that makes profit + `quantity` DECIMAL(16, 8) UNSIGNED NOT NULL, + + -- trade_amount is the quote quantity of the trade that makes profit + `trade_amount` DECIMAL(16, 8) UNSIGNED NOT NULL, + + `traded_at` DATETIME(3) NOT NULL, + + -- fee + `fee_in_usd` DECIMAL(16, 8) UNSIGNED NOT NULL, + `fee` DECIMAL(16, 8) UNSIGNED NOT NULL, + `fee_currency` VARCHAR(10) NOT NULL, PRIMARY KEY (`gid`), - UNIQUE KEY `trade_id` (`trade_id`) ); diff --git a/migrations/sqlite3/20220304153309_add_profit_table.sql b/migrations/sqlite3/20220304153309_add_profit_table.sql index 73eab6853..8c2d99456 100644 --- a/migrations/sqlite3/20220304153309_add_profit_table.sql +++ b/migrations/sqlite3/20220304153309_add_profit_table.sql @@ -1,33 +1,60 @@ -- +up CREATE TABLE `profits` ( - `gid` INTEGER PRIMARY KEY AUTOINCREMENT, + `gid` INTEGER PRIMARY KEY AUTOINCREMENT, - `exchange` VARCHAR(24) NOT NULL DEFAULT '', - - `symbol` VARCHAR(8) NOT NULL, - - `trade_id` INTEGER NOT NULL, + `symbol` VARCHAR(8) NOT NULL, -- average_cost is the position average cost - `average_cost` DECIMAL(16, 8) NOT NULL, + `average_cost` DECIMAL(16, 8) NOT NULL, -- profit is the pnl (profit and loss) - `profit` DECIMAL(16, 8) NOT NULL, + `profit` DECIMAL(16, 8) NOT NULL, - -- price is the price of the trade that makes profit - `price` DECIMAL(16, 8) NOT NULL, + -- net_profit is the pnl (profit and loss) + `net_profit` DECIMAL(16, 8) NOT NULL, - -- quantity is the quantity of the trade that makes profit - `quantity` DECIMAL(16, 8) NOT NULL, + -- profit_margin is the pnl (profit and loss) + `profit_margin` DECIMAL(16, 8) NOT NULL, - -- quote_quantity is the quote quantity of the trade that makes profit - `quote_quantity` DECIMAL(16, 8) NOT NULL, + -- net_profit_margin is the pnl (profit and loss) + `net_profit_margin` DECIMAL(16, 8) NOT NULL, + + `quote_currency` VARCHAR(10) NOT NULL, + + `base_currency` VARCHAR(10) NOT NULL, + + -- ------------------------------------------------------- + -- embedded trade data -- + -- ------------------------------------------------------- + `exchange` VARCHAR(24) NOT NULL DEFAULT '', + + `is_futures` BOOLEAN NOT NULL DEFAULT FALSE, + + `is_margin` BOOLEAN NOT NULL DEFAULT FALSE, + + `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE, + + `trade_id` BIGINT NOT NULL, -- side is the side of the trade that makes profit - `side` VARCHAR(4) NOT NULL DEFAULT '', + `side` VARCHAR(4) NOT NULL DEFAULT '', - `traded_at` DATETIME(3) NOT NULL + -- price is the price of the trade that makes profit + `price` DECIMAL(16, 8) NOT NULL, + + -- quantity is the quantity of the trade that makes profit + `quantity` DECIMAL(16, 8) NOT NULL, + + -- trade_amount is the quote quantity of the trade that makes profit + `trade_amount` DECIMAL(16, 8) NOT NULL, + + `traded_at` DATETIME(3) NOT NULL, + + -- fee + `fee_in_usd` DECIMAL(16, 8) NOT NULL, + `fee` DECIMAL(16, 8) NOT NULL, + `fee_currency` VARCHAR(10) NOT NULL ); -- +down From 09dea3938d065ce7fa6c986c2b46bebc7accd640 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 4 Mar 2022 19:24:40 +0800 Subject: [PATCH 05/32] implement profit insert --- .../mysql/20220304153317_add_profit_table.sql | 53 +++++++++------- .../20220304153309_add_profit_table.sql | 51 +++++++++------- pkg/service/profit.go | 61 ++++++++++++++++--- pkg/types/profit.go | 10 ++- 4 files changed, 117 insertions(+), 58 deletions(-) diff --git a/migrations/mysql/20220304153317_add_profit_table.sql b/migrations/mysql/20220304153317_add_profit_table.sql index 5db882b66..f74a21304 100644 --- a/migrations/mysql/20220304153317_add_profit_table.sql +++ b/migrations/mysql/20220304153317_add_profit_table.sql @@ -1,60 +1,67 @@ -- +up CREATE TABLE `profits` ( - `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, - `symbol` VARCHAR(8) NOT NULL, + `strategy` VARCHAR(32) NOT NULL, + `strategy_instance_id` VARCHAR(64) NOT NULL, + + `symbol` VARCHAR(8) NOT NULL, -- average_cost is the position average cost - `average_cost` DECIMAL(16, 8) UNSIGNED NOT NULL, + `average_cost` DECIMAL(16, 8) UNSIGNED NOT NULL, -- profit is the pnl (profit and loss) - `profit` DECIMAL(16, 8) NOT NULL, + `profit` DECIMAL(16, 8) NOT NULL, -- net_profit is the pnl (profit and loss) - `net_profit` DECIMAL(16, 8) NOT NULL, + `net_profit` DECIMAL(16, 8) NOT NULL, -- profit_margin is the pnl (profit and loss) - `profit_margin` DECIMAL(16, 8) NOT NULL, + `profit_margin` DECIMAL(16, 8) NOT NULL, -- net_profit_margin is the pnl (profit and loss) - `net_profit_margin` DECIMAL(16, 8) NOT NULL, + `net_profit_margin` DECIMAL(16, 8) NOT NULL, - `quote_currency` VARCHAR(10) NOT NULL, + `quote_currency` VARCHAR(10) NOT NULL, - `base_currency` VARCHAR(10) NOT NULL, + `base_currency` VARCHAR(10) NOT NULL, -- ------------------------------------------------------- -- embedded trade data -- -- ------------------------------------------------------- - `exchange` VARCHAR(24) NOT NULL DEFAULT '', + `exchange` VARCHAR(24) NOT NULL DEFAULT '', - `is_futures` BOOLEAN NOT NULL DEFAULT FALSE, + `is_futures` BOOLEAN NOT NULL DEFAULT FALSE, - `is_margin` BOOLEAN NOT NULL DEFAULT FALSE, + `is_margin` BOOLEAN NOT NULL DEFAULT FALSE, - `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE, + `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE, - `trade_id` BIGINT UNSIGNED NOT NULL, + `trade_id` BIGINT UNSIGNED NOT NULL, -- side is the side of the trade that makes profit - `side` VARCHAR(4) NOT NULL DEFAULT '', + `side` VARCHAR(4) NOT NULL DEFAULT '', + + `is_buyer` BOOLEAN NOT NULL DEFAULT FALSE, + + `is_maker` BOOLEAN NOT NULL DEFAULT FALSE, -- price is the price of the trade that makes profit - `price` DECIMAL(16, 8) UNSIGNED NOT NULL, + `price` DECIMAL(16, 8) UNSIGNED NOT NULL, -- quantity is the quantity of the trade that makes profit - `quantity` DECIMAL(16, 8) UNSIGNED NOT NULL, + `quantity` DECIMAL(16, 8) UNSIGNED NOT NULL, - -- trade_amount is the quote quantity of the trade that makes profit - `trade_amount` DECIMAL(16, 8) UNSIGNED NOT NULL, + -- quote_quantity is the quote quantity of the trade that makes profit + `quote_quantity` DECIMAL(16, 8) UNSIGNED NOT NULL, - `traded_at` DATETIME(3) NOT NULL, + `traded_at` DATETIME(3) NOT NULL, -- fee - `fee_in_usd` DECIMAL(16, 8) UNSIGNED NOT NULL, - `fee` DECIMAL(16, 8) UNSIGNED NOT NULL, - `fee_currency` VARCHAR(10) NOT NULL, + `fee_in_usd` DECIMAL(16, 8) UNSIGNED, + `fee` DECIMAL(16, 8) UNSIGNED NOT NULL, + `fee_currency` VARCHAR(10) NOT NULL, PRIMARY KEY (`gid`), UNIQUE KEY `trade_id` (`trade_id`) diff --git a/migrations/sqlite3/20220304153309_add_profit_table.sql b/migrations/sqlite3/20220304153309_add_profit_table.sql index 8c2d99456..290d32539 100644 --- a/migrations/sqlite3/20220304153309_add_profit_table.sql +++ b/migrations/sqlite3/20220304153309_add_profit_table.sql @@ -1,60 +1,67 @@ -- +up CREATE TABLE `profits` ( - `gid` INTEGER PRIMARY KEY AUTOINCREMENT, + `gid` INTEGER PRIMARY KEY AUTOINCREMENT, - `symbol` VARCHAR(8) NOT NULL, + `strategy` VARCHAR(32) NOT NULL, + `strategy_instance_id` VARCHAR(64) NOT NULL, + + `symbol` VARCHAR(8) NOT NULL, -- average_cost is the position average cost - `average_cost` DECIMAL(16, 8) NOT NULL, + `average_cost` DECIMAL(16, 8) NOT NULL, -- profit is the pnl (profit and loss) - `profit` DECIMAL(16, 8) NOT NULL, + `profit` DECIMAL(16, 8) NOT NULL, -- net_profit is the pnl (profit and loss) - `net_profit` DECIMAL(16, 8) NOT NULL, + `net_profit` DECIMAL(16, 8) NOT NULL, -- profit_margin is the pnl (profit and loss) - `profit_margin` DECIMAL(16, 8) NOT NULL, + `profit_margin` DECIMAL(16, 8) NOT NULL, -- net_profit_margin is the pnl (profit and loss) - `net_profit_margin` DECIMAL(16, 8) NOT NULL, + `net_profit_margin` DECIMAL(16, 8) NOT NULL, - `quote_currency` VARCHAR(10) NOT NULL, + `quote_currency` VARCHAR(10) NOT NULL, - `base_currency` VARCHAR(10) NOT NULL, + `base_currency` VARCHAR(10) NOT NULL, -- ------------------------------------------------------- -- embedded trade data -- -- ------------------------------------------------------- - `exchange` VARCHAR(24) NOT NULL DEFAULT '', + `exchange` VARCHAR(24) NOT NULL DEFAULT '', - `is_futures` BOOLEAN NOT NULL DEFAULT FALSE, + `is_futures` BOOLEAN NOT NULL DEFAULT FALSE, - `is_margin` BOOLEAN NOT NULL DEFAULT FALSE, + `is_margin` BOOLEAN NOT NULL DEFAULT FALSE, - `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE, + `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE, - `trade_id` BIGINT NOT NULL, + `trade_id` BIGINT NOT NULL, -- side is the side of the trade that makes profit - `side` VARCHAR(4) NOT NULL DEFAULT '', + `side` VARCHAR(4) NOT NULL DEFAULT '', + + `is_buyer` BOOLEAN NOT NULL DEFAULT FALSE, + + `is_maker` BOOLEAN NOT NULL DEFAULT FALSE, -- price is the price of the trade that makes profit - `price` DECIMAL(16, 8) NOT NULL, + `price` DECIMAL(16, 8) NOT NULL, -- quantity is the quantity of the trade that makes profit - `quantity` DECIMAL(16, 8) NOT NULL, + `quantity` DECIMAL(16, 8) NOT NULL, -- trade_amount is the quote quantity of the trade that makes profit - `trade_amount` DECIMAL(16, 8) NOT NULL, + `quote_quantity` DECIMAL(16, 8) NOT NULL, - `traded_at` DATETIME(3) NOT NULL, + `traded_at` DATETIME(3) NOT NULL, -- fee - `fee_in_usd` DECIMAL(16, 8) NOT NULL, - `fee` DECIMAL(16, 8) NOT NULL, - `fee_currency` VARCHAR(10) NOT NULL + `fee_in_usd` DECIMAL(16, 8), + `fee` DECIMAL(16, 8) NOT NULL, + `fee_currency` VARCHAR(10) NOT NULL ); -- +down diff --git a/pkg/service/profit.go b/pkg/service/profit.go index 7920d3aca..4baf1029e 100644 --- a/pkg/service/profit.go +++ b/pkg/service/profit.go @@ -37,23 +37,64 @@ func (s *ProfitService) Load(ctx context.Context, id int64) (*types.Trade, error return nil, errors.Wrapf(ErrTradeNotFound, "trade id:%d not found", id) } -func (s *ProfitService) scanRows(rows *sqlx.Rows) (trades []types.Trade, err error) { +func (s *ProfitService) scanRows(rows *sqlx.Rows) (profits []types.Profit, err error) { for rows.Next() { - var trade types.Trade - if err := rows.StructScan(&trade); err != nil { - return trades, err + var profit types.Profit + if err := rows.StructScan(&profit); err != nil { + return profits, err } - trades = append(trades, trade) + profits = append(profits, profit) } - return trades, rows.Err() + return profits, rows.Err() } -func (s *ProfitService) Insert(trade types.Trade) error { +func (s *ProfitService) Insert(profit types.Profit) error { _, err := s.DB.NamedExec(` - INSERT INTO profits (id, exchange, symbol, trade_id, average_cost, profit, price, quantity, quote_quantity, side, traded_at, is_margin, is_futures, is_isolated) - VALUES (:id, :exchange, :order_id, :symbol, :price, :quantity, :quote_quantity, :side, :is_buyer, :is_maker, :fee, :fee_currency, :traded_at, :is_margin, :is_futures, :is_isolated)`, - trade) + INSERT INTO profits ( + strategy, + strategy_instance_id, + symbol, + average_cost, + profit, + trade_id, + price, + quantity, + quote_quantity, + side, + is_buyer, + is_maker, + fee, + fee_currency, + fee_in_usd, + traded_at, + exchange, + is_margin, + is_futures, + is_isolated + ) VALUES ( + :strategy, + :strategy_instance_id, + :symbol, + :average_cost, + :profit, + :trade_id, + :price, + :quantity, + :quote_quantity, + :side, + :is_buyer, + :is_maker, + :fee, + :fee_currency, + :fee_in_usd, + :traded_at, + :exchange, + :is_margin, + :is_futures, + :is_isolated + )`, + profit) return err } diff --git a/pkg/types/profit.go b/pkg/types/profit.go index 90d06134d..1f54019c0 100644 --- a/pkg/types/profit.go +++ b/pkg/types/profit.go @@ -20,8 +20,7 @@ type Profit struct { // NetProfit is (profit - trading fee) NetProfit fixedpoint.Value `json:"netProfit" db:"net_profit"` AverageCost fixedpoint.Value `json:"averageCost" db:"average_ost"` - - TradeAmount fixedpoint.Value `json:"tradeAmount" db:"trade_amount"` + TradeAmount fixedpoint.Value `json:"tradeAmount" db:"quote_quantity"` // ProfitMargin is a percentage of the profit and the capital amount ProfitMargin fixedpoint.Value `json:"profitMargin" db:"profit_margin"` @@ -32,10 +31,15 @@ type Profit struct { QuoteCurrency string `json:"quoteCurrency" db:"quote_currency"` BaseCurrency string `json:"baseCurrency" db:"base_currency"` + IsBuyer bool `json:"isBuyer" db:"is_buyer"` + IsMaker bool `json:"isMaker" db:"is_maker"` + // FeeInUSD is the summed fee of this profit, // you will need to convert the trade fee into USD since the fee currencies can be different. FeeInUSD fixedpoint.Value `json:"feeInUSD" db:"fee_in_usd"` - Time time.Time `json:"time" db:"time"` + Fee fixedpoint.Value `json:"fee" db:"fee"` + FeeCurrency string `json:"feeCurrency" db:"fee_currency"` + Time time.Time `json:"tradedAt" db:"traded_at"` Strategy string `json:"strategy" db:"strategy"` StrategyInstanceID string `json:"strategyInstanceID" db:"strategy_instance_id"` } From a642aa1a5adc435813e3fd53860bac22cc32d061 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 00:27:44 +0800 Subject: [PATCH 06/32] service: add more columns --- pkg/service/profit.go | 28 +++++++++++++++++--------- pkg/service/profit_test.go | 40 ++++++++++++++++++++++++++++++++++++++ pkg/types/profit.go | 18 ++++++++++++----- 3 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 pkg/service/profit_test.go diff --git a/pkg/service/profit.go b/pkg/service/profit.go index 4baf1029e..15af10261 100644 --- a/pkg/service/profit.go +++ b/pkg/service/profit.go @@ -54,31 +54,41 @@ func (s *ProfitService) Insert(profit types.Profit) error { _, err := s.DB.NamedExec(` INSERT INTO profits ( strategy, - strategy_instance_id, + strategy_instance_id, symbol, + quote_currency, + base_currency, average_cost, profit, + net_profit, + profit_margin, + net_profit_margin, trade_id, price, quantity, quote_quantity, side, - is_buyer, - is_maker, - fee, - fee_currency, - fee_in_usd, + is_buyer, + is_maker, + fee, + fee_currency, + fee_in_usd, traded_at, exchange, is_margin, is_futures, is_isolated ) VALUES ( - :strategy, + :strategy, :strategy_instance_id, :symbol, + :quote_currency, + :base_currency, :average_cost, - :profit, + :profit, + :net_profit, + :profit_margin, + :net_profit_margin, :trade_id, :price, :quantity, @@ -88,7 +98,7 @@ func (s *ProfitService) Insert(profit types.Profit) error { :is_maker, :fee, :fee_currency, - :fee_in_usd, + :fee_in_usd, :traded_at, :exchange, :is_margin, diff --git a/pkg/service/profit_test.go b/pkg/service/profit_test.go new file mode 100644 index 000000000..c26b7a296 --- /dev/null +++ b/pkg/service/profit_test.go @@ -0,0 +1,40 @@ +package service + +import ( + "testing" + "time" + + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func TestProfitService(t *testing.T) { + db, err := prepareDB(t) + if err != nil { + t.Fatal(err) + } + + defer db.Close() + + xdb := sqlx.NewDb(db.DB, "sqlite3") + service := &ProfitService{DB: xdb} + + err = service.Insert(types.Profit{ + Symbol: "BTCUSDT", + BaseCurrency: "BTC", + QuoteCurrency: "USDT", + Profit: fixedpoint.NewFromFloat(1.01), + NetProfit: fixedpoint.NewFromFloat(0.98), + AverageCost: fixedpoint.NewFromFloat(44000), + TradeID: 99, + Price: fixedpoint.NewFromFloat(44300), + Quantity: fixedpoint.NewFromFloat(0.001), + TradeAmount: fixedpoint.NewFromFloat(44.0), + Exchange: types.ExchangeMax, + Time: time.Now(), + }) + assert.NoError(t, err) +} diff --git a/pkg/types/profit.go b/pkg/types/profit.go index 1f54019c0..0a4401704 100644 --- a/pkg/types/profit.go +++ b/pkg/types/profit.go @@ -19,8 +19,7 @@ type Profit struct { // NetProfit is (profit - trading fee) NetProfit fixedpoint.Value `json:"netProfit" db:"net_profit"` - AverageCost fixedpoint.Value `json:"averageCost" db:"average_ost"` - TradeAmount fixedpoint.Value `json:"tradeAmount" db:"quote_quantity"` + AverageCost fixedpoint.Value `json:"averageCost" db:"average_cost"` // ProfitMargin is a percentage of the profit and the capital amount ProfitMargin fixedpoint.Value `json:"profitMargin" db:"profit_margin"` @@ -31,9 +30,13 @@ type Profit struct { QuoteCurrency string `json:"quoteCurrency" db:"quote_currency"` BaseCurrency string `json:"baseCurrency" db:"base_currency"` - IsBuyer bool `json:"isBuyer" db:"is_buyer"` - IsMaker bool `json:"isMaker" db:"is_maker"` - + TradeID uint64 `json:"tradeID" db:"trade_id"` + Side string `json:"side" db:"side"` + IsBuyer bool `json:"isBuyer" db:"is_buyer"` + IsMaker bool `json:"isMaker" db:"is_maker"` + Price fixedpoint.Value `json:"price" db:"price"` + Quantity fixedpoint.Value `json:"quantity"` + TradeAmount fixedpoint.Value `json:"quoteQuantity" db:"quote_quantity"` // FeeInUSD is the summed fee of this profit, // you will need to convert the trade fee into USD since the fee currencies can be different. FeeInUSD fixedpoint.Value `json:"feeInUSD" db:"fee_in_usd"` @@ -42,6 +45,11 @@ type Profit struct { Time time.Time `json:"tradedAt" db:"traded_at"` Strategy string `json:"strategy" db:"strategy"` StrategyInstanceID string `json:"strategyInstanceID" db:"strategy_instance_id"` + + Exchange ExchangeName `json:"exchange" db:"exchange"` + IsMargin bool `json:"isMargin" db:"is_margin"` + IsFutures bool `json:"isFutures" db:"is_futures"` + IsIsolated bool `json:"isIsolated" db:"is_isolated"` } func (p *Profit) SlackAttachment() slack.Attachment { From 82e5520ee48566e422f2ea42f9c789fc51e24814 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 00:28:13 +0800 Subject: [PATCH 07/32] service: update profit service tests --- pkg/service/profit_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/service/profit_test.go b/pkg/service/profit_test.go index c26b7a296..731ebc10a 100644 --- a/pkg/service/profit_test.go +++ b/pkg/service/profit_test.go @@ -26,9 +26,9 @@ func TestProfitService(t *testing.T) { Symbol: "BTCUSDT", BaseCurrency: "BTC", QuoteCurrency: "USDT", + AverageCost: fixedpoint.NewFromFloat(44000), Profit: fixedpoint.NewFromFloat(1.01), NetProfit: fixedpoint.NewFromFloat(0.98), - AverageCost: fixedpoint.NewFromFloat(44000), TradeID: 99, Price: fixedpoint.NewFromFloat(44300), Quantity: fixedpoint.NewFromFloat(0.001), From 197d750cb410561e7be538ada6a237d711140596 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 01:39:53 +0800 Subject: [PATCH 08/32] all: update profit struct fields --- pkg/service/profit_test.go | 7 ++-- pkg/strategy/bollmaker/strategy.go | 4 +-- pkg/strategy/xmaker/strategy.go | 4 +-- pkg/types/position.go | 34 ++++++++++++++++++ pkg/types/profit.go | 57 +++++++++++++++++------------- 5 files changed, 75 insertions(+), 31 deletions(-) diff --git a/pkg/service/profit_test.go b/pkg/service/profit_test.go index 731ebc10a..e4616a80e 100644 --- a/pkg/service/profit_test.go +++ b/pkg/service/profit_test.go @@ -23,18 +23,19 @@ func TestProfitService(t *testing.T) { service := &ProfitService{DB: xdb} err = service.Insert(types.Profit{ - Symbol: "BTCUSDT", + Symbol: "BTCUSDT", BaseCurrency: "BTC", QuoteCurrency: "USDT", AverageCost: fixedpoint.NewFromFloat(44000), Profit: fixedpoint.NewFromFloat(1.01), NetProfit: fixedpoint.NewFromFloat(0.98), TradeID: 99, + Side: types.SideTypeSell, Price: fixedpoint.NewFromFloat(44300), Quantity: fixedpoint.NewFromFloat(0.001), - TradeAmount: fixedpoint.NewFromFloat(44.0), + QuoteQuantity: fixedpoint.NewFromFloat(44.0), Exchange: types.ExchangeMax, - Time: time.Now(), + TradedAt: time.Now(), }) assert.NoError(t, err) } diff --git a/pkg/strategy/bollmaker/strategy.go b/pkg/strategy/bollmaker/strategy.go index bdf4f9a1c..893d44b1e 100644 --- a/pkg/strategy/bollmaker/strategy.go +++ b/pkg/strategy/bollmaker/strategy.go @@ -575,12 +575,12 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se Symbol: s.Symbol, Profit: profit, NetProfit: netProfit, - TradeAmount: trade.QuoteQuantity, + QuoteQuantity: trade.QuoteQuantity, ProfitMargin: profit.Div(trade.QuoteQuantity), NetProfitMargin: netProfit.Div(trade.QuoteQuantity), QuoteCurrency: s.state.Position.QuoteCurrency, BaseCurrency: s.state.Position.BaseCurrency, - Time: trade.Time.Time(), + TradedAt: trade.Time.Time(), } s.state.ProfitStats.AddProfit(p) s.Notify(&p) diff --git a/pkg/strategy/xmaker/strategy.go b/pkg/strategy/xmaker/strategy.go index cf85e0d9d..6a02a9be5 100644 --- a/pkg/strategy/xmaker/strategy.go +++ b/pkg/strategy/xmaker/strategy.go @@ -767,12 +767,12 @@ func (s *Strategy) CrossRun(ctx context.Context, orderExecutionRouter bbgo.Order Symbol: s.Symbol, Profit: profit, NetProfit: netProfit, - TradeAmount: trade.QuoteQuantity, + QuoteQuantity: trade.QuoteQuantity, ProfitMargin: profit.Div(trade.QuoteQuantity), NetProfitMargin: netProfit.Div(trade.QuoteQuantity), QuoteCurrency: s.state.Position.QuoteCurrency, BaseCurrency: s.state.Position.BaseCurrency, - Time: trade.Time.Time(), + TradedAt: trade.Time.Time(), } s.state.ProfitStats.AddProfit(p) s.Notify(&p) diff --git a/pkg/types/position.go b/pkg/types/position.go index e922ed98a..d876a8e3c 100644 --- a/pkg/types/position.go +++ b/pkg/types/position.go @@ -53,6 +53,40 @@ type Position struct { sync.Mutex } +// NewProfit generates the profit object from the current position +func (p *Position) NewProfit(profit, netProfit fixedpoint.Value, trade Trade) *Profit { + return &Profit{ + Symbol: p.Symbol, + QuoteCurrency: p.QuoteCurrency, + BaseCurrency: p.BaseCurrency, + AverageCost: p.AverageCost, + + // profit related fields + Profit: profit, + NetProfit: netProfit, + ProfitMargin: profit.Div(trade.QuoteQuantity), + NetProfitMargin: netProfit.Div(trade.QuoteQuantity), + + // trade related fields + TradeID: trade.ID, + Price: trade.Price, + Quantity: trade.Quantity, + QuoteQuantity: trade.QuoteQuantity, + IsMaker: trade.IsMaker, + IsBuyer: trade.IsBuyer, + Side: trade.Side, + + Fee: trade.Fee, + FeeCurrency: trade.FeeCurrency, + + TradedAt: trade.Time.Time(), + + IsFutures: trade.IsFutures, + IsMargin: trade.IsMargin, + IsIsolated: trade.IsIsolated, + } +} + func (p *Position) NewClosePositionOrder(percentage fixedpoint.Value) *SubmitOrder { base := p.GetBase() quantity := base.Mul(percentage) diff --git a/pkg/types/profit.go b/pkg/types/profit.go index 0a4401704..8b8ef783b 100644 --- a/pkg/types/profit.go +++ b/pkg/types/profit.go @@ -12,14 +12,21 @@ import ( // Profit struct stores the PnL information type Profit struct { - Symbol string `json:"symbol"` + // --- position related fields + // ------------------------------------------- + // Symbol is the symbol of the position + Symbol string `json:"symbol"` + QuoteCurrency string `json:"quoteCurrency" db:"quote_currency"` + BaseCurrency string `json:"baseCurrency" db:"base_currency"` + AverageCost fixedpoint.Value `json:"averageCost" db:"average_cost"` + // profit related fields + // ------------------------------------------- // Profit is the profit of this trade made. negative profit means loss. Profit fixedpoint.Value `json:"profit" db:"profit"` // NetProfit is (profit - trading fee) - NetProfit fixedpoint.Value `json:"netProfit" db:"net_profit"` - AverageCost fixedpoint.Value `json:"averageCost" db:"average_cost"` + NetProfit fixedpoint.Value `json:"netProfit" db:"net_profit"` // ProfitMargin is a percentage of the profit and the capital amount ProfitMargin fixedpoint.Value `json:"profitMargin" db:"profit_margin"` @@ -27,29 +34,31 @@ type Profit struct { // NetProfitMargin is a percentage of the net profit and the capital amount NetProfitMargin fixedpoint.Value `json:"netProfitMargin" db:"net_profit_margin"` - QuoteCurrency string `json:"quoteCurrency" db:"quote_currency"` - BaseCurrency string `json:"baseCurrency" db:"base_currency"` + // trade related fields + // -------------------------------------------- + // TradeID is the exchange trade id of that trade + TradeID uint64 `json:"tradeID" db:"trade_id"` + Side SideType `json:"side" db:"side"` + IsBuyer bool `json:"isBuyer" db:"is_buyer"` + IsMaker bool `json:"isMaker" db:"is_maker"` + Price fixedpoint.Value `json:"price" db:"price"` + Quantity fixedpoint.Value `json:"quantity" db:"quantity"` + QuoteQuantity fixedpoint.Value `json:"quoteQuantity" db:"quote_quantity"` - TradeID uint64 `json:"tradeID" db:"trade_id"` - Side string `json:"side" db:"side"` - IsBuyer bool `json:"isBuyer" db:"is_buyer"` - IsMaker bool `json:"isMaker" db:"is_maker"` - Price fixedpoint.Value `json:"price" db:"price"` - Quantity fixedpoint.Value `json:"quantity"` - TradeAmount fixedpoint.Value `json:"quoteQuantity" db:"quote_quantity"` // FeeInUSD is the summed fee of this profit, // you will need to convert the trade fee into USD since the fee currencies can be different. - FeeInUSD fixedpoint.Value `json:"feeInUSD" db:"fee_in_usd"` - Fee fixedpoint.Value `json:"fee" db:"fee"` - FeeCurrency string `json:"feeCurrency" db:"fee_currency"` - Time time.Time `json:"tradedAt" db:"traded_at"` - Strategy string `json:"strategy" db:"strategy"` - StrategyInstanceID string `json:"strategyInstanceID" db:"strategy_instance_id"` + FeeInUSD fixedpoint.Value `json:"feeInUSD" db:"fee_in_usd"` + Fee fixedpoint.Value `json:"fee" db:"fee"` + FeeCurrency string `json:"feeCurrency" db:"fee_currency"` + Exchange ExchangeName `json:"exchange" db:"exchange"` + IsMargin bool `json:"isMargin" db:"is_margin"` + IsFutures bool `json:"isFutures" db:"is_futures"` + IsIsolated bool `json:"isIsolated" db:"is_isolated"` + TradedAt time.Time `json:"tradedAt" db:"traded_at"` - Exchange ExchangeName `json:"exchange" db:"exchange"` - IsMargin bool `json:"isMargin" db:"is_margin"` - IsFutures bool `json:"isFutures" db:"is_futures"` - IsIsolated bool `json:"isIsolated" db:"is_isolated"` + // strategy related fields + Strategy string `json:"strategy" db:"strategy"` + StrategyInstanceID string `json:"strategyInstanceID" db:"strategy_instance_id"` } func (p *Profit) SlackAttachment() slack.Attachment { @@ -84,10 +93,10 @@ func (p *Profit) SlackAttachment() slack.Attachment { }) } - if !p.TradeAmount.IsZero() { + if !p.QuoteQuantity.IsZero() { fields = append(fields, slack.AttachmentField{ Title: "Trade Amount", - Value: p.TradeAmount.String() + " " + p.QuoteCurrency, + Value: p.QuoteQuantity.String() + " " + p.QuoteCurrency, Short: true, }) } From 5fe0b699277f2f7b610ebd67398911889c7c3a7c Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 01:41:23 +0800 Subject: [PATCH 09/32] bollmaker: use the new profit generator method --- pkg/strategy/bollmaker/strategy.go | 12 +----------- pkg/types/position.go | 4 ++-- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/pkg/strategy/bollmaker/strategy.go b/pkg/strategy/bollmaker/strategy.go index 893d44b1e..765ef0776 100644 --- a/pkg/strategy/bollmaker/strategy.go +++ b/pkg/strategy/bollmaker/strategy.go @@ -571,17 +571,7 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se s.tradeCollector = bbgo.NewTradeCollector(s.Symbol, s.state.Position, s.orderStore) s.tradeCollector.OnProfit(func(trade types.Trade, profit fixedpoint.Value, netProfit fixedpoint.Value) { log.Infof("generated profit: %v", profit) - p := types.Profit{ - Symbol: s.Symbol, - Profit: profit, - NetProfit: netProfit, - QuoteQuantity: trade.QuoteQuantity, - ProfitMargin: profit.Div(trade.QuoteQuantity), - NetProfitMargin: netProfit.Div(trade.QuoteQuantity), - QuoteCurrency: s.state.Position.QuoteCurrency, - BaseCurrency: s.state.Position.BaseCurrency, - TradedAt: trade.Time.Time(), - } + p := s.state.Position.NewProfit(trade, profit, netProfit) s.state.ProfitStats.AddProfit(p) s.Notify(&p) s.Notify(&s.state.ProfitStats) diff --git a/pkg/types/position.go b/pkg/types/position.go index d876a8e3c..d3ba675e6 100644 --- a/pkg/types/position.go +++ b/pkg/types/position.go @@ -54,8 +54,8 @@ type Position struct { } // NewProfit generates the profit object from the current position -func (p *Position) NewProfit(profit, netProfit fixedpoint.Value, trade Trade) *Profit { - return &Profit{ +func (p *Position) NewProfit(trade Trade, profit, netProfit fixedpoint.Value) Profit { + return Profit{ Symbol: p.Symbol, QuoteCurrency: p.QuoteCurrency, BaseCurrency: p.BaseCurrency, From db4d8a31bc6f8e5c25197c5b687c8e6f7d93ffaa Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 02:33:25 +0800 Subject: [PATCH 10/32] bbgo: implement parseStructAndInject --- pkg/bbgo/trader.go | 54 +++++++++++++++++++++++++++++++++++++ pkg/bbgo/trader_test.go | 60 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 pkg/bbgo/trader_test.go diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index 8f8f6d60a..0c0633343 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -413,3 +413,57 @@ func (trader *Trader) injectCommonServices(rs reflect.Value) error { func (trader *Trader) ReportPnL() *PnLReporterManager { return NewPnLReporter(&trader.environment.Notifiability) } + +func parseStructAndInject(f interface{}, objects ...interface{}) error { + sv := reflect.ValueOf(f) + st := reflect.TypeOf(f) + + if st.Kind() != reflect.Ptr { + return fmt.Errorf("f needs to be a pointer of a struct, %s given", st) + } + + // solve the pointer + st = st.Elem() + sv = sv.Elem() + + if st.Kind() != reflect.Struct { + return fmt.Errorf("f needs to be a struct, %s given", st) + } + + for i := 0; i < sv.NumField(); i++ { + fv := sv.Field(i) + ft := fv.Type() + + switch k := fv.Kind(); k { + + case reflect.Ptr, reflect.Struct: + for oi := 0; oi < len(objects); oi++ { + obj := objects[oi] + ot := reflect.TypeOf(obj) + if ft.AssignableTo(ot) { + if !fv.CanSet() { + return fmt.Errorf("field %v of %s can not be set to %s", fv, sv.Type(), ot) + } + fv.Set(reflect.ValueOf(obj)) + } + } + + case reflect.Interface: + for oi := 0; oi < len(objects); oi++ { + obj := objects[oi] + objT := reflect.TypeOf(obj) + log.Debugln( + ft.PkgPath(), + ft.Name(), + objT, "implements", ft, "=", objT.Implements(ft), + ) + + if objT.Implements(ft) { + fv.Set(reflect.ValueOf(obj)) + } + } + } + } + + return nil +} diff --git a/pkg/bbgo/trader_test.go b/pkg/bbgo/trader_test.go new file mode 100644 index 000000000..6bb5ea095 --- /dev/null +++ b/pkg/bbgo/trader_test.go @@ -0,0 +1,60 @@ +package bbgo + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/types" +) + +func Test_parseStructAndInject(t *testing.T) { + t.Run("pointer", func(t *testing.T) { + ss := struct { + a int + Env *Environment + }{ + a: 1, + Env: nil, + } + err := parseStructAndInject(&ss, &Environment{}) + assert.NoError(t, err) + assert.NotNil(t, ss.Env) + }) + t.Run("struct", func(t *testing.T) { + ss := struct { + a int + Env Environment + }{ + a: 1, + } + err := parseStructAndInject(&ss, Environment{ + startTime: time.Now(), + }) + assert.NoError(t, err) + assert.NotEqual(t, time.Time{}, ss.Env.startTime) + }) + t.Run("interface/any", func(t *testing.T) { + ss := struct { + Any interface{} // anything + }{ + Any: nil, + } + err := parseStructAndInject(&ss, &Environment{ + startTime: time.Now(), + }) + assert.NoError(t, err) + assert.NotNil(t, ss.Any) + }) + t.Run("interface/stringer", func(t *testing.T) { + ss := struct { + Stringer types.Stringer // stringer interface + }{ + Stringer: nil, + } + err := parseStructAndInject(&ss, &types.Trade{}) + assert.NoError(t, err) + assert.NotNil(t, ss.Stringer) + }) +} From fa7bab2c3aa7c9bde8bd4ab557f7c3d4f2a1683a Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 02:51:43 +0800 Subject: [PATCH 11/32] bbgo: improve dynamic injection --- pkg/bbgo/trader.go | 80 ++++++++++++++++------------------------- pkg/bbgo/trader_test.go | 12 +++++++ 2 files changed, 43 insertions(+), 49 deletions(-) diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index 0c0633343..b46c317d0 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -219,6 +219,16 @@ func (trader *Trader) RunSingleExchangeStrategy(ctx context.Context, strategy Si return errors.New("strategy object is not a struct") } + if err := parseStructAndInject(strategy, + &trader.Graceful, + &trader.logger, + trader.environment.Notifiability, + trader.environment.TradeService, + trader.environment.AccountService, + ) ; err != nil { + return err + } + if err := trader.injectCommonServices(rs); err != nil { return err } @@ -229,36 +239,24 @@ func (trader *Trader) RunSingleExchangeStrategy(ctx context.Context, strategy Si if symbol, ok := isSymbolBasedStrategy(rs); ok { log.Infof("found symbol based strategy from %s", rs.Type()) - if _, ok := hasField(rs, "Market"); ok { - if market, ok := session.Market(symbol); ok { - // let's make the market object passed by pointer - if err := injectField(rs, "Market", &market, false); err != nil { - return errors.Wrapf(err, "failed to inject Market on %T", strategy) - } - } + + market, ok := session.Market(symbol) + if !ok { + return fmt.Errorf("market of symbol %s not found", symbol) } - // StandardIndicatorSet - if _, ok := hasField(rs, "StandardIndicatorSet"); ok { - indicatorSet, ok := session.StandardIndicatorSet(symbol) - if !ok { - return fmt.Errorf("standardIndicatorSet of symbol %s not found", symbol) - } - - if err := injectField(rs, "StandardIndicatorSet", indicatorSet, true); err != nil { - return errors.Wrapf(err, "failed to inject StandardIndicatorSet on %T", strategy) - } + indicatorSet, ok := session.StandardIndicatorSet(symbol) + if !ok { + return fmt.Errorf("standardIndicatorSet of symbol %s not found", symbol) } - if _, ok := hasField(rs, "MarketDataStore"); ok { - store, ok := session.MarketDataStore(symbol) - if !ok { - return fmt.Errorf("marketDataStore of symbol %s not found", symbol) - } + store, ok := session.MarketDataStore(symbol) + if !ok { + return fmt.Errorf("marketDataStore of symbol %s not found", symbol) + } - if err := injectField(rs, "MarketDataStore", store, true); err != nil { - return errors.Wrapf(err, "failed to inject MarketDataStore on %T", strategy) - } + if err := parseStructAndInject(strategy, market, indicatorSet, store) ; err != nil { + return errors.Wrapf(err, "failed to inject object into %T", strategy) } } @@ -357,30 +355,6 @@ func (trader *Trader) Run(ctx context.Context) error { } func (trader *Trader) injectCommonServices(rs reflect.Value) error { - if err := injectField(rs, "Graceful", &trader.Graceful, true); err != nil { - return errors.Wrap(err, "failed to inject Graceful") - } - - if err := injectField(rs, "Logger", &trader.logger, false); err != nil { - return errors.Wrap(err, "failed to inject Logger") - } - - if err := injectField(rs, "Notifiability", &trader.environment.Notifiability, false); err != nil { - return errors.Wrap(err, "failed to inject Notifiability") - } - - if trader.environment.TradeService != nil { - if err := injectField(rs, "TradeService", trader.environment.TradeService, true); err != nil { - return errors.Wrap(err, "failed to inject TradeService") - } - } - - if trader.environment.AccountService != nil { - if err := injectField(rs, "AccountService", trader.environment.AccountService, true); err != nil { - return errors.Wrap(err, "failed to inject AccountService") - } - } - if field, ok := hasField(rs, "Persistence"); ok { if trader.environment.PersistenceServiceFacade == nil { log.Warnf("strategy has Persistence field but persistence service is not defined") @@ -439,6 +413,10 @@ func parseStructAndInject(f interface{}, objects ...interface{}) error { case reflect.Ptr, reflect.Struct: for oi := 0; oi < len(objects); oi++ { obj := objects[oi] + if obj == nil { + continue + } + ot := reflect.TypeOf(obj) if ft.AssignableTo(ot) { if !fv.CanSet() { @@ -451,6 +429,10 @@ func parseStructAndInject(f interface{}, objects ...interface{}) error { case reflect.Interface: for oi := 0; oi < len(objects); oi++ { obj := objects[oi] + if obj == nil { + continue + } + objT := reflect.TypeOf(obj) log.Debugln( ft.PkgPath(), diff --git a/pkg/bbgo/trader_test.go b/pkg/bbgo/trader_test.go index 6bb5ea095..8ee3d0712 100644 --- a/pkg/bbgo/trader_test.go +++ b/pkg/bbgo/trader_test.go @@ -10,6 +10,18 @@ import ( ) func Test_parseStructAndInject(t *testing.T) { + t.Run("skip nil", func(t *testing.T) { + ss := struct { + a int + Env *Environment + }{ + a: 1, + Env: nil, + } + err := parseStructAndInject(&ss, nil) + assert.NoError(t, err) + assert.Nil(t, ss.Env) + }) t.Run("pointer", func(t *testing.T) { ss := struct { a int From cd6b37ac3b16c2c5c1537d89f2cc90dd88cf7b52 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 03:19:45 +0800 Subject: [PATCH 12/32] bbgo: skip unexported fields for injection --- pkg/bbgo/trader.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index b46c317d0..c4ba7e018 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -225,7 +225,7 @@ func (trader *Trader) RunSingleExchangeStrategy(ctx context.Context, strategy Si trader.environment.Notifiability, trader.environment.TradeService, trader.environment.AccountService, - ) ; err != nil { + ); err != nil { return err } @@ -255,7 +255,7 @@ func (trader *Trader) RunSingleExchangeStrategy(ctx context.Context, strategy Si return fmt.Errorf("marketDataStore of symbol %s not found", symbol) } - if err := parseStructAndInject(strategy, market, indicatorSet, store) ; err != nil { + if err := parseStructAndInject(strategy, market, indicatorSet, store); err != nil { return errors.Wrapf(err, "failed to inject object into %T", strategy) } } @@ -408,6 +408,11 @@ func parseStructAndInject(f interface{}, objects ...interface{}) error { fv := sv.Field(i) ft := fv.Type() + // skip unexported fields + if !st.Field(i).IsExported() { + continue + } + switch k := fv.Kind(); k { case reflect.Ptr, reflect.Struct: @@ -420,7 +425,7 @@ func parseStructAndInject(f interface{}, objects ...interface{}) error { ot := reflect.TypeOf(obj) if ft.AssignableTo(ot) { if !fv.CanSet() { - return fmt.Errorf("field %v of %s can not be set to %s", fv, sv.Type(), ot) + return fmt.Errorf("field %v of %s can not be set to %s, make sure it is an exported field", fv, sv.Type(), ot) } fv.Set(reflect.ValueOf(obj)) } From a6053e0e592dd1c2675b6b20b1354636786ba688 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 03:20:20 +0800 Subject: [PATCH 13/32] bbgo: move inject function to injection.go --- pkg/bbgo/injection.go | 69 +++++++++++++++++++++++++++++++++++++++++++ pkg/bbgo/trader.go | 66 ----------------------------------------- 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/pkg/bbgo/injection.go b/pkg/bbgo/injection.go index 5d2338412..25f4b8bbd 100644 --- a/pkg/bbgo/injection.go +++ b/pkg/bbgo/injection.go @@ -3,6 +3,8 @@ package bbgo import ( "fmt" "reflect" + + "github.com/sirupsen/logrus" ) func isSymbolBasedStrategy(rs reflect.Value) (string, bool) { @@ -53,3 +55,70 @@ func injectField(rs reflect.Value, fieldName string, obj interface{}, pointerOnl return nil } + +func parseStructAndInject(f interface{}, objects ...interface{}) error { + sv := reflect.ValueOf(f) + st := reflect.TypeOf(f) + + if st.Kind() != reflect.Ptr { + return fmt.Errorf("f needs to be a pointer of a struct, %s given", st) + } + + // solve the pointer + st = st.Elem() + sv = sv.Elem() + + if st.Kind() != reflect.Struct { + return fmt.Errorf("f needs to be a struct, %s given", st) + } + + for i := 0; i < sv.NumField(); i++ { + fv := sv.Field(i) + ft := fv.Type() + + // skip unexported fields + if !st.Field(i).IsExported() { + continue + } + + switch k := fv.Kind(); k { + + case reflect.Ptr, reflect.Struct: + for oi := 0; oi < len(objects); oi++ { + obj := objects[oi] + if obj == nil { + continue + } + + ot := reflect.TypeOf(obj) + if ft.AssignableTo(ot) { + if !fv.CanSet() { + return fmt.Errorf("field %v of %s can not be set to %s, make sure it is an exported field", fv, sv.Type(), ot) + } + fv.Set(reflect.ValueOf(obj)) + } + } + + case reflect.Interface: + for oi := 0; oi < len(objects); oi++ { + obj := objects[oi] + if obj == nil { + continue + } + + objT := reflect.TypeOf(obj) + logrus.Debugln( + ft.PkgPath(), + ft.Name(), + objT, "implements", ft, "=", objT.Implements(ft), + ) + + if objT.Implements(ft) { + fv.Set(reflect.ValueOf(obj)) + } + } + } + } + + return nil +} diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index c4ba7e018..597cda08d 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -388,69 +388,3 @@ func (trader *Trader) ReportPnL() *PnLReporterManager { return NewPnLReporter(&trader.environment.Notifiability) } -func parseStructAndInject(f interface{}, objects ...interface{}) error { - sv := reflect.ValueOf(f) - st := reflect.TypeOf(f) - - if st.Kind() != reflect.Ptr { - return fmt.Errorf("f needs to be a pointer of a struct, %s given", st) - } - - // solve the pointer - st = st.Elem() - sv = sv.Elem() - - if st.Kind() != reflect.Struct { - return fmt.Errorf("f needs to be a struct, %s given", st) - } - - for i := 0; i < sv.NumField(); i++ { - fv := sv.Field(i) - ft := fv.Type() - - // skip unexported fields - if !st.Field(i).IsExported() { - continue - } - - switch k := fv.Kind(); k { - - case reflect.Ptr, reflect.Struct: - for oi := 0; oi < len(objects); oi++ { - obj := objects[oi] - if obj == nil { - continue - } - - ot := reflect.TypeOf(obj) - if ft.AssignableTo(ot) { - if !fv.CanSet() { - return fmt.Errorf("field %v of %s can not be set to %s, make sure it is an exported field", fv, sv.Type(), ot) - } - fv.Set(reflect.ValueOf(obj)) - } - } - - case reflect.Interface: - for oi := 0; oi < len(objects); oi++ { - obj := objects[oi] - if obj == nil { - continue - } - - objT := reflect.TypeOf(obj) - log.Debugln( - ft.PkgPath(), - ft.Name(), - objT, "implements", ft, "=", objT.Implements(ft), - ) - - if objT.Implements(ft) { - fv.Set(reflect.ValueOf(obj)) - } - } - } - } - - return nil -} From 47023729ecacec27e8a70aff2abc5459fb8c8d06 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 12:39:39 +0800 Subject: [PATCH 14/32] bbgo: rewrite field injection --- pkg/bbgo/injection.go | 2 +- pkg/bbgo/trader.go | 61 ++++++++++++++++--------------------------- 2 files changed, 23 insertions(+), 40 deletions(-) diff --git a/pkg/bbgo/injection.go b/pkg/bbgo/injection.go index 25f4b8bbd..a339db7e3 100644 --- a/pkg/bbgo/injection.go +++ b/pkg/bbgo/injection.go @@ -64,7 +64,7 @@ func parseStructAndInject(f interface{}, objects ...interface{}) error { return fmt.Errorf("f needs to be a pointer of a struct, %s given", st) } - // solve the pointer + // solve the reference st = st.Elem() sv = sv.Elem() diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index 597cda08d..3a68eb8a7 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -219,17 +219,7 @@ func (trader *Trader) RunSingleExchangeStrategy(ctx context.Context, strategy Si return errors.New("strategy object is not a struct") } - if err := parseStructAndInject(strategy, - &trader.Graceful, - &trader.logger, - trader.environment.Notifiability, - trader.environment.TradeService, - trader.environment.AccountService, - ); err != nil { - return err - } - - if err := trader.injectCommonServices(rs); err != nil { + if err := trader.injectCommonServices(strategy) ; err != nil { return err } @@ -337,12 +327,11 @@ func (trader *Trader) Run(ctx context.Context) error { // get the struct element from the struct pointer rs = rs.Elem() - if rs.Kind() != reflect.Struct { continue } - if err := trader.injectCommonServices(rs); err != nil { + if err := trader.injectCommonServices(strategy) ; err != nil { return err } @@ -354,37 +343,31 @@ func (trader *Trader) Run(ctx context.Context) error { return trader.environment.Connect(ctx) } -func (trader *Trader) injectCommonServices(rs reflect.Value) error { - if field, ok := hasField(rs, "Persistence"); ok { - if trader.environment.PersistenceServiceFacade == nil { - log.Warnf("strategy has Persistence field but persistence service is not defined") - } else { - if field.IsNil() { - field.Set(reflect.ValueOf(&Persistence{ - PersistenceSelector: &PersistenceSelector{ - StoreID: "default", - Type: "memory", - }, - Facade: trader.environment.PersistenceServiceFacade, - })) - } else { - elem := field.Elem() - if elem.Kind() != reflect.Struct { - return fmt.Errorf("field Persistence is not a struct element") - } - - if err := injectField(elem, "Facade", trader.environment.PersistenceServiceFacade, true); err != nil { - return errors.Wrap(err, "failed to inject Persistence") - } - } - } +func (trader *Trader) injectCommonServices(s interface{}) error { + defaultPersistenceSelector := &PersistenceSelector{ + StoreID: "default", + Type: "memory", } - return nil + persistenceFacade := trader.environment.PersistenceServiceFacade + persistence := &Persistence{ + PersistenceSelector: defaultPersistenceSelector, + Facade: persistenceFacade, + } + + return parseStructAndInject(s, + &trader.Graceful, + &trader.logger, + trader.environment.Notifiability, + trader.environment.TradeService, + trader.environment.AccountService, + persistence, + persistenceFacade, + ) } + // ReportPnL configure and set the PnLReporter with the given notifier func (trader *Trader) ReportPnL() *PnLReporterManager { return NewPnLReporter(&trader.environment.Notifiability) } - From a9f9fa8fedbb828481e45fd9e9e74487419016ba Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 12:40:56 +0800 Subject: [PATCH 15/32] bollmaker: add Environment field and Market field for injection --- pkg/strategy/bollmaker/strategy.go | 46 +++++++++++++----------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/pkg/strategy/bollmaker/strategy.go b/pkg/strategy/bollmaker/strategy.go index 765ef0776..2b669effa 100644 --- a/pkg/strategy/bollmaker/strategy.go +++ b/pkg/strategy/bollmaker/strategy.go @@ -52,7 +52,9 @@ type Strategy struct { *bbgo.Notifiability *bbgo.Persistence + Environment *bbgo.Environment StandardIndicatorSet *bbgo.StandardIndicatorSet + Market types.Market // Symbol is the market symbol you want to trade Symbol string `json:"symbol"` @@ -141,8 +143,7 @@ type Strategy struct { bbgo.SmartStops session *bbgo.ExchangeSession - book *types.StreamOrderBook - market types.Market + book *types.StreamOrderBook state *State @@ -214,8 +215,8 @@ func (s *Strategy) ClosePosition(ctx context.Context, percentage fixedpoint.Valu side = types.SideTypeSell } - if quantity.Compare(s.market.MinQuantity) < 0 { - return fmt.Errorf("order quantity %v is too small, less than %v", quantity, s.market.MinQuantity) + if quantity.Compare(s.Market.MinQuantity) < 0 { + return fmt.Errorf("order quantity %v is too small, less than %v", quantity, s.Market.MinQuantity) } submitOrder := types.SubmitOrder{ @@ -223,7 +224,7 @@ func (s *Strategy) ClosePosition(ctx context.Context, percentage fixedpoint.Valu Side: side, Type: types.OrderTypeMarket, Quantity: quantity, - Market: s.market, + Market: s.Market, } s.Notify("Submitting %s %s order to close position by %v", s.Symbol, side.String(), percentage, submitOrder) @@ -264,13 +265,13 @@ func (s *Strategy) LoadState() error { // if position is nil, we need to allocate a new position for calculation if s.state.Position == nil { - s.state.Position = types.NewPositionFromMarket(s.market) + s.state.Position = types.NewPositionFromMarket(s.Market) } // init profit states - s.state.ProfitStats.Symbol = s.market.Symbol - s.state.ProfitStats.BaseCurrency = s.market.BaseCurrency - s.state.ProfitStats.QuoteCurrency = s.market.QuoteCurrency + s.state.ProfitStats.Symbol = s.Market.Symbol + s.state.ProfitStats.BaseCurrency = s.Market.BaseCurrency + s.state.ProfitStats.QuoteCurrency = s.Market.QuoteCurrency if s.state.ProfitStats.AccumulatedSince == 0 { s.state.ProfitStats.AccumulatedSince = time.Now().Unix() } @@ -323,7 +324,7 @@ func (s *Strategy) placeOrders(ctx context.Context, orderExecutor bbgo.OrderExec Type: types.OrderTypeLimitMaker, Quantity: sellQuantity, Price: askPrice, - Market: s.market, + Market: s.Market, GroupID: s.groupID, } buyOrder := types.SubmitOrder{ @@ -332,14 +333,14 @@ func (s *Strategy) placeOrders(ctx context.Context, orderExecutor bbgo.OrderExec Type: types.OrderTypeLimitMaker, Quantity: buyQuantity, Price: bidPrice, - Market: s.market, + Market: s.Market, GroupID: s.groupID, } var submitOrders []types.SubmitOrder - baseBalance, hasBaseBalance := balances[s.market.BaseCurrency] - quoteBalance, hasQuoteBalance := balances[s.market.QuoteCurrency] + baseBalance, hasBaseBalance := balances[s.Market.BaseCurrency] + quoteBalance, hasQuoteBalance := balances[s.Market.QuoteCurrency] downBand := s.defaultBoll.LastDownBand() upBand := s.defaultBoll.LastUpBand() @@ -421,12 +422,12 @@ func (s *Strategy) placeOrders(ctx context.Context, orderExecutor bbgo.OrderExec case UpTrend: skew := s.UptrendSkew - buyOrder.Quantity = fixedpoint.Max(s.market.MinQuantity, sellOrder.Quantity.Mul(skew)) + buyOrder.Quantity = fixedpoint.Max(s.Market.MinQuantity, sellOrder.Quantity.Mul(skew)) case DownTrend: skew := s.DowntrendSkew ratio := fixedpoint.One.Div(skew) - sellOrder.Quantity = fixedpoint.Max(s.market.MinQuantity, buyOrder.Quantity.Mul(ratio)) + sellOrder.Quantity = fixedpoint.Max(s.Market.MinQuantity, buyOrder.Quantity.Mul(ratio)) } @@ -506,12 +507,12 @@ func (s *Strategy) detectPriceTrend(inc *indicator.BOLL, price float64) PriceTre } func (s *Strategy) adjustOrderQuantity(submitOrder types.SubmitOrder) types.SubmitOrder { - if submitOrder.Quantity.Mul(submitOrder.Price).Compare(s.market.MinNotional) < 0 { - submitOrder.Quantity = bbgo.AdjustFloatQuantityByMinAmount(submitOrder.Quantity, submitOrder.Price, s.market.MinNotional.Mul(notionModifier)) + if submitOrder.Quantity.Mul(submitOrder.Price).Compare(s.Market.MinNotional) < 0 { + submitOrder.Quantity = bbgo.AdjustFloatQuantityByMinAmount(submitOrder.Quantity, submitOrder.Price, s.Market.MinNotional.Mul(notionModifier)) } - if submitOrder.Quantity.Compare(s.market.MinQuantity) < 0 { - submitOrder.Quantity = fixedpoint.Max(submitOrder.Quantity, s.market.MinQuantity) + if submitOrder.Quantity.Compare(s.Market.MinQuantity) < 0 { + submitOrder.Quantity = fixedpoint.Max(submitOrder.Quantity, s.Market.MinQuantity) } return submitOrder @@ -540,13 +541,6 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se // initial required information s.session = session - - market, ok := session.Market(s.Symbol) - if !ok { - return fmt.Errorf("market %s not found", s.Symbol) - } - s.market = market - s.neutralBoll = s.StandardIndicatorSet.BOLL(s.NeutralBollinger.IntervalWindow, s.NeutralBollinger.BandWidth) s.defaultBoll = s.StandardIndicatorSet.BOLL(s.DefaultBollinger.IntervalWindow, s.DefaultBollinger.BandWidth) From bdcae5b76348feb9b5bca38dbc83a4b7964d102d Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 12:49:53 +0800 Subject: [PATCH 16/32] bbgo: add more injection types --- pkg/bbgo/trader.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index 3a68eb8a7..707c0dd80 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -245,7 +245,15 @@ func (trader *Trader) RunSingleExchangeStrategy(ctx context.Context, strategy Si return fmt.Errorf("marketDataStore of symbol %s not found", symbol) } - if err := parseStructAndInject(strategy, market, indicatorSet, store); err != nil { + + + if err := parseStructAndInject(strategy, + market, + indicatorSet, + store, + session, + session.OrderExecutor, + ); err != nil { return errors.Wrapf(err, "failed to inject object into %T", strategy) } } From c1ac738ca0a9d4179b3bc94641c78982449f040e Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 12:59:47 +0800 Subject: [PATCH 17/32] bbgo: add doc comment for parseStructAndInject --- pkg/bbgo/environment.go | 29 +++++++++++++++++------------ pkg/bbgo/injection.go | 11 ++++++++++- pkg/bbgo/trader.go | 2 -- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index 658d848d1..0765a5eb5 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -584,7 +584,7 @@ func (environ *Environment) SyncSession(ctx context.Context, session *ExchangeSe } func (environ *Environment) syncSession(ctx context.Context, session *ExchangeSession, defaultSymbols ...string) error { - symbols, err := getSessionSymbols(session, defaultSymbols...) + symbols, err := session.getSessionSymbols(defaultSymbols...) if err != nil { return err } @@ -594,17 +594,6 @@ func (environ *Environment) syncSession(ctx context.Context, session *ExchangeSe return environ.SyncService.SyncSessionSymbols(ctx, session.Exchange, environ.syncStartTime, symbols...) } -func getSessionSymbols(session *ExchangeSession, defaultSymbols ...string) ([]string, error) { - if session.IsolatedMargin { - return []string{session.IsolatedMarginSymbol}, nil - } - - if len(defaultSymbols) > 0 { - return defaultSymbols, nil - } - - return session.FindPossibleSymbols() -} func (environ *Environment) ConfigureNotificationSystem(userConfig *Config) error { environ.Notifiability = Notifiability{ @@ -654,6 +643,10 @@ func (environ *Environment) ConfigureNotificationSystem(userConfig *Config) erro return nil } +// getAuthStoreID returns the authentication store id +// if telegram bot token is defined, the bot id will be used. +// if not, env var $USER will be used. +// if both are not defined, a default "default" will be used. func getAuthStoreID() string { telegramBotToken := viper.GetString("telegram-bot-token") if len(telegramBotToken) > 0 { @@ -924,3 +917,15 @@ And then enter your token `, token) } + +func (session *ExchangeSession) getSessionSymbols(defaultSymbols ...string) ([]string, error) { + if session.IsolatedMargin { + return []string{session.IsolatedMarginSymbol}, nil + } + + if len(defaultSymbols) > 0 { + return defaultSymbols, nil + } + + return session.FindPossibleSymbols() +} diff --git a/pkg/bbgo/injection.go b/pkg/bbgo/injection.go index a339db7e3..03ff9fd7f 100644 --- a/pkg/bbgo/injection.go +++ b/pkg/bbgo/injection.go @@ -56,6 +56,9 @@ func injectField(rs reflect.Value, fieldName string, obj interface{}, pointerOnl return nil } +// parseStructAndInject parses the struct fields and injects the objects into the corresponding fields by its type. +// if the given object is a reference of an object, the type of the target field MUST BE a pointer field. +// if the given object is a struct value, the type of the target field CAN BE a pointer field or a struct value field. func parseStructAndInject(f interface{}, objects ...interface{}) error { sv := reflect.ValueOf(f) st := reflect.TypeOf(f) @@ -95,7 +98,13 @@ func parseStructAndInject(f interface{}, objects ...interface{}) error { if !fv.CanSet() { return fmt.Errorf("field %v of %s can not be set to %s, make sure it is an exported field", fv, sv.Type(), ot) } - fv.Set(reflect.ValueOf(obj)) + + if k == reflect.Ptr && ot.Kind() == reflect.Struct { + fv.Set(reflect.ValueOf(obj).Addr()) + } else { + fv.Set(reflect.ValueOf(obj)) + } + } } diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index 707c0dd80..ec35fb230 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -245,8 +245,6 @@ func (trader *Trader) RunSingleExchangeStrategy(ctx context.Context, strategy Si return fmt.Errorf("marketDataStore of symbol %s not found", symbol) } - - if err := parseStructAndInject(strategy, market, indicatorSet, From 1f1c26a9e5943b2d75b748cac1cabcbf2a024fa0 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 13:37:27 +0800 Subject: [PATCH 18/32] bbgo: inject more service objects --- pkg/bbgo/trader.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index ec35fb230..a1abb9e91 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -366,6 +366,8 @@ func (trader *Trader) injectCommonServices(s interface{}) error { &trader.logger, trader.environment.Notifiability, trader.environment.TradeService, + trader.environment.OrderService, + trader.environment.DatabaseService, trader.environment.AccountService, persistence, persistenceFacade, From 35b0d8dc0dcc0a820f6105014db9471fa0d82abc Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 5 Mar 2022 13:40:20 +0800 Subject: [PATCH 19/32] bbgo: add profit service to environment --- pkg/bbgo/environment.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index 0765a5eb5..6aa5c97a6 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -76,6 +76,7 @@ type Environment struct { DatabaseService *service.DatabaseService OrderService *service.OrderService TradeService *service.TradeService + ProfitService *service.ProfitService BacktestService *service.BacktestService RewardService *service.RewardService SyncService *service.SyncService @@ -170,6 +171,7 @@ func (environ *Environment) ConfigureDatabaseDriver(ctx context.Context, driver environ.TradeService = &service.TradeService{DB: db} environ.RewardService = &service.RewardService{DB: db} environ.AccountService = &service.AccountService{DB: db} + environ.ProfitService = &service.ProfitService{DB: db} environ.SyncService = &service.SyncService{ TradeService: environ.TradeService, @@ -569,6 +571,17 @@ func (environ *Environment) Sync(ctx context.Context, userConfig ...*Config) err return nil } +func (environ *Environment) RecordProfit(strategyID, strategyInstanceID string, profit types.Profit) { + if environ.DatabaseService == nil { + return + } + if environ.ProfitService == nil { + return + } + + +} + func (environ *Environment) SyncSession(ctx context.Context, session *ExchangeSession, defaultSymbols ...string) error { if environ.SyncService == nil { return nil From 3a15738fec7e717c911ae1bd2b06e7d7ee91e77f Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 14:06:19 +0800 Subject: [PATCH 20/32] pull out default persistence selector --- pkg/bbgo/trader.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index a1abb9e91..3790a24ca 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -349,12 +349,12 @@ func (trader *Trader) Run(ctx context.Context) error { return trader.environment.Connect(ctx) } -func (trader *Trader) injectCommonServices(s interface{}) error { - defaultPersistenceSelector := &PersistenceSelector{ - StoreID: "default", - Type: "memory", - } +var defaultPersistenceSelector = &PersistenceSelector{ + StoreID: "default", + Type: "memory", +} +func (trader *Trader) injectCommonServices(s interface{}) error { persistenceFacade := trader.environment.PersistenceServiceFacade persistence := &Persistence{ PersistenceSelector: defaultPersistenceSelector, From f6ec2e78e625e3a400b420ca912e169bfa2b4324 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 15:37:41 +0800 Subject: [PATCH 21/32] record profits --- pkg/bbgo/environment.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index 6aa5c97a6..14c385b40 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -571,7 +571,7 @@ func (environ *Environment) Sync(ctx context.Context, userConfig ...*Config) err return nil } -func (environ *Environment) RecordProfit(strategyID, strategyInstanceID string, profit types.Profit) { +func (environ *Environment) RecordProfit(profit types.Profit) { if environ.DatabaseService == nil { return } @@ -579,7 +579,9 @@ func (environ *Environment) RecordProfit(strategyID, strategyInstanceID string, return } - + if err := environ.ProfitService.Insert(profit) ; err != nil { + log.WithError(err).Errorf("can not insert profit record: %+v", profit) + } } func (environ *Environment) SyncSession(ctx context.Context, session *ExchangeSession, defaultSymbols ...string) error { From 8fa0e6702ca19cb568a451f35c5b8a1e448006f3 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 15:38:58 +0800 Subject: [PATCH 22/32] bollmaker: assign strategy id and instance id --- pkg/strategy/bollmaker/strategy.go | 3 +++ pkg/types/position.go | 34 ++++++++++++++---------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/pkg/strategy/bollmaker/strategy.go b/pkg/strategy/bollmaker/strategy.go index 2b669effa..9a514c634 100644 --- a/pkg/strategy/bollmaker/strategy.go +++ b/pkg/strategy/bollmaker/strategy.go @@ -566,6 +566,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se s.tradeCollector.OnProfit(func(trade types.Trade, profit fixedpoint.Value, netProfit fixedpoint.Value) { log.Infof("generated profit: %v", profit) p := s.state.Position.NewProfit(trade, profit, netProfit) + p.Strategy = ID + p.StrategyInstanceID = instanceID + s.state.ProfitStats.AddProfit(p) s.Notify(&p) s.Notify(&s.state.ProfitStats) diff --git a/pkg/types/position.go b/pkg/types/position.go index d3ba675e6..875ba801c 100644 --- a/pkg/types/position.go +++ b/pkg/types/position.go @@ -60,30 +60,28 @@ func (p *Position) NewProfit(trade Trade, profit, netProfit fixedpoint.Value) Pr QuoteCurrency: p.QuoteCurrency, BaseCurrency: p.BaseCurrency, AverageCost: p.AverageCost, - // profit related fields Profit: profit, NetProfit: netProfit, ProfitMargin: profit.Div(trade.QuoteQuantity), NetProfitMargin: netProfit.Div(trade.QuoteQuantity), - // trade related fields - TradeID: trade.ID, - Price: trade.Price, - Quantity: trade.Quantity, - QuoteQuantity: trade.QuoteQuantity, - IsMaker: trade.IsMaker, - IsBuyer: trade.IsBuyer, - Side: trade.Side, - - Fee: trade.Fee, - FeeCurrency: trade.FeeCurrency, - - TradedAt: trade.Time.Time(), - - IsFutures: trade.IsFutures, - IsMargin: trade.IsMargin, - IsIsolated: trade.IsIsolated, + TradeID: trade.ID, + Side: trade.Side, + IsBuyer: trade.IsBuyer, + IsMaker: trade.IsMaker, + Price: trade.Price, + Quantity: trade.Quantity, + QuoteQuantity: trade.QuoteQuantity, + // FeeInUSD: 0, + Fee: trade.Fee, + FeeCurrency: trade.FeeCurrency, + + Exchange: trade.Exchange, + IsMargin: trade.IsMargin, + IsFutures: trade.IsFutures, + IsIsolated: trade.IsIsolated, + TradedAt: trade.Time.Time(), } } From 25f3aeef584582fe8deef21f6eed2f9043021706 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 15:39:20 +0800 Subject: [PATCH 23/32] bollmaker: call RecordProfit --- pkg/strategy/bollmaker/strategy.go | 1 + pkg/types/position.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/strategy/bollmaker/strategy.go b/pkg/strategy/bollmaker/strategy.go index 9a514c634..22d0ba941 100644 --- a/pkg/strategy/bollmaker/strategy.go +++ b/pkg/strategy/bollmaker/strategy.go @@ -568,6 +568,7 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se p := s.state.Position.NewProfit(trade, profit, netProfit) p.Strategy = ID p.StrategyInstanceID = instanceID + s.Environment.RecordProfit(p) s.state.ProfitStats.AddProfit(p) s.Notify(&p) diff --git a/pkg/types/position.go b/pkg/types/position.go index 875ba801c..68c8429ee 100644 --- a/pkg/types/position.go +++ b/pkg/types/position.go @@ -76,7 +76,7 @@ func (p *Position) NewProfit(trade Trade, profit, netProfit fixedpoint.Value) Pr // FeeInUSD: 0, Fee: trade.Fee, FeeCurrency: trade.FeeCurrency, - + Exchange: trade.Exchange, IsMargin: trade.IsMargin, IsFutures: trade.IsFutures, From b1ba5386b3b05185187fda5797895e777698cddd Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 16:09:15 +0800 Subject: [PATCH 24/32] fix bbgo.Notifiability injection --- pkg/bbgo/injection_test.go | 75 +++++++++++++++++++++++++++++ pkg/bbgo/trader.go | 2 +- pkg/bbgo/trader_test.go | 70 --------------------------- pkg/strategy/pricealert/strategy.go | 2 +- 4 files changed, 77 insertions(+), 72 deletions(-) diff --git a/pkg/bbgo/injection_test.go b/pkg/bbgo/injection_test.go index 1bb2fe6f8..5d656991e 100644 --- a/pkg/bbgo/injection_test.go +++ b/pkg/bbgo/injection_test.go @@ -3,10 +3,12 @@ package bbgo import ( "reflect" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/c9s/bbgo/pkg/service" + "github.com/c9s/bbgo/pkg/types" ) func Test_injectField(t *testing.T) { @@ -28,3 +30,76 @@ func Test_injectField(t *testing.T) { err := injectField(rv, "TradeService", ts, true) assert.NoError(t, err) } + +func Test_parseStructAndInject(t *testing.T) { + t.Run("skip nil", func(t *testing.T) { + ss := struct { + a int + Env *Environment + }{ + a: 1, + Env: nil, + } + err := parseStructAndInject(&ss, nil) + assert.NoError(t, err) + assert.Nil(t, ss.Env) + }) + t.Run("pointer", func(t *testing.T) { + ss := struct { + a int + Env *Environment + }{ + a: 1, + Env: nil, + } + err := parseStructAndInject(&ss, &Environment{}) + assert.NoError(t, err) + assert.NotNil(t, ss.Env) + }) + + t.Run("composition", func(t *testing.T) { + type TT struct { + *service.TradeService + } + ss := &TT{} + err := parseStructAndInject(&ss, &service.TradeService{}) + assert.NoError(t, err) + assert.NotNil(t, ss.TradeService) + }) + + t.Run("struct", func(t *testing.T) { + ss := struct { + a int + Env Environment + }{ + a: 1, + } + err := parseStructAndInject(&ss, Environment{ + startTime: time.Now(), + }) + assert.NoError(t, err) + assert.NotEqual(t, time.Time{}, ss.Env.startTime) + }) + t.Run("interface/any", func(t *testing.T) { + ss := struct { + Any interface{} // anything + }{ + Any: nil, + } + err := parseStructAndInject(&ss, &Environment{ + startTime: time.Now(), + }) + assert.NoError(t, err) + assert.NotNil(t, ss.Any) + }) + t.Run("interface/stringer", func(t *testing.T) { + ss := struct { + Stringer types.Stringer // stringer interface + }{ + Stringer: nil, + } + err := parseStructAndInject(&ss, &types.Trade{}) + assert.NoError(t, err) + assert.NotNil(t, ss.Stringer) + }) +} diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index 3790a24ca..62daefe2d 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -364,7 +364,7 @@ func (trader *Trader) injectCommonServices(s interface{}) error { return parseStructAndInject(s, &trader.Graceful, &trader.logger, - trader.environment.Notifiability, + &trader.environment.Notifiability, trader.environment.TradeService, trader.environment.OrderService, trader.environment.DatabaseService, diff --git a/pkg/bbgo/trader_test.go b/pkg/bbgo/trader_test.go index 8ee3d0712..920078f66 100644 --- a/pkg/bbgo/trader_test.go +++ b/pkg/bbgo/trader_test.go @@ -1,72 +1,2 @@ package bbgo -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/c9s/bbgo/pkg/types" -) - -func Test_parseStructAndInject(t *testing.T) { - t.Run("skip nil", func(t *testing.T) { - ss := struct { - a int - Env *Environment - }{ - a: 1, - Env: nil, - } - err := parseStructAndInject(&ss, nil) - assert.NoError(t, err) - assert.Nil(t, ss.Env) - }) - t.Run("pointer", func(t *testing.T) { - ss := struct { - a int - Env *Environment - }{ - a: 1, - Env: nil, - } - err := parseStructAndInject(&ss, &Environment{}) - assert.NoError(t, err) - assert.NotNil(t, ss.Env) - }) - t.Run("struct", func(t *testing.T) { - ss := struct { - a int - Env Environment - }{ - a: 1, - } - err := parseStructAndInject(&ss, Environment{ - startTime: time.Now(), - }) - assert.NoError(t, err) - assert.NotEqual(t, time.Time{}, ss.Env.startTime) - }) - t.Run("interface/any", func(t *testing.T) { - ss := struct { - Any interface{} // anything - }{ - Any: nil, - } - err := parseStructAndInject(&ss, &Environment{ - startTime: time.Now(), - }) - assert.NoError(t, err) - assert.NotNil(t, ss.Any) - }) - t.Run("interface/stringer", func(t *testing.T) { - ss := struct { - Stringer types.Stringer // stringer interface - }{ - Stringer: nil, - } - err := parseStructAndInject(&ss, &types.Trade{}) - assert.NoError(t, err) - assert.NotNil(t, ss.Stringer) - }) -} diff --git a/pkg/strategy/pricealert/strategy.go b/pkg/strategy/pricealert/strategy.go index 345ac4d54..0e484e52b 100644 --- a/pkg/strategy/pricealert/strategy.go +++ b/pkg/strategy/pricealert/strategy.go @@ -16,7 +16,7 @@ func init() { type Strategy struct { // The notification system will be injected into the strategy automatically. - bbgo.Notifiability + *bbgo.Notifiability // These fields will be filled from the config file (it translates YAML to JSON) Symbol string `json:"symbol"` From 099d860c5a84bccabe98e384d96313758d2102a9 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 18:26:36 +0800 Subject: [PATCH 25/32] fix: fix Test_parseStructAndInject test --- pkg/bbgo/injection_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bbgo/injection_test.go b/pkg/bbgo/injection_test.go index 5d656991e..dd6370320 100644 --- a/pkg/bbgo/injection_test.go +++ b/pkg/bbgo/injection_test.go @@ -61,7 +61,7 @@ func Test_parseStructAndInject(t *testing.T) { type TT struct { *service.TradeService } - ss := &TT{} + ss := TT{} err := parseStructAndInject(&ss, &service.TradeService{}) assert.NoError(t, err) assert.NotNil(t, ss.TradeService) From 917684aa278f5742ba8647c544de23e1f2c14e1a Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 18:17:42 +0800 Subject: [PATCH 26/32] bbgo: inject environment object --- pkg/bbgo/trader.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index 62daefe2d..f5d1736de 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -369,6 +369,7 @@ func (trader *Trader) injectCommonServices(s interface{}) error { trader.environment.OrderService, trader.environment.DatabaseService, trader.environment.AccountService, + trader.environment, persistence, persistenceFacade, ) From f3577a4182575b4d3f64cf514aa5ca756d75aec6 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 18:25:16 +0800 Subject: [PATCH 27/32] fix: if it's an empty time, do not return a driver value --- pkg/types/time.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/types/time.go b/pkg/types/time.go index fb4951334..47ef9ff72 100644 --- a/pkg/types/time.go +++ b/pkg/types/time.go @@ -176,6 +176,9 @@ func NewTimeFromUnix(sec int64, nsec int64) Time { // Value implements the driver.Valuer interface // see http://jmoiron.net/blog/built-in-interfaces/ func (t Time) Value() (driver.Value, error) { + if time.Time(t) == (time.Time{}) { + return nil, nil + } return time.Time(t), nil } From af2070b9082e1b665540e62f1569692f07b484b7 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 18:32:33 +0800 Subject: [PATCH 28/32] binance: add updated time field --- pkg/exchange/binance/parse.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/exchange/binance/parse.go b/pkg/exchange/binance/parse.go index b2d922848..2f88a1db2 100644 --- a/pkg/exchange/binance/parse.go +++ b/pkg/exchange/binance/parse.go @@ -102,7 +102,6 @@ type ExecutionReportEvent struct { } func (e *ExecutionReportEvent) Order() (*types.Order, error) { - switch e.CurrentExecutionType { case "NEW", "CANCELED", "REJECTED", "EXPIRED": case "REPLACED": @@ -113,7 +112,6 @@ func (e *ExecutionReportEvent) Order() (*types.Order, error) { orderCreationTime := time.Unix(0, e.OrderCreationTime*int64(time.Millisecond)) return &types.Order{ - Exchange: types.ExchangeBinance, SubmitOrder: types.SubmitOrder{ Symbol: e.Symbol, ClientOrderID: e.ClientOrderID, @@ -123,10 +121,13 @@ func (e *ExecutionReportEvent) Order() (*types.Order, error) { Price: e.OrderPrice, TimeInForce: types.TimeInForce(e.TimeInForce), }, + Exchange: types.ExchangeBinance, + IsWorking: e.IsOnBook, OrderID: uint64(e.OrderID), Status: toGlobalOrderStatus(binance.OrderStatusType(e.CurrentOrderStatus)), ExecutedQuantity: e.CumulativeFilledQuantity, CreationTime: types.Time(orderCreationTime), + UpdateTime: types.Time(orderCreationTime), }, nil } From 586013d9f2a234046948a76cc0b790125d77e077 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 18:33:21 +0800 Subject: [PATCH 29/32] max: fix order update message --- pkg/exchange/max/convert.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/exchange/max/convert.go b/pkg/exchange/max/convert.go index a973c49e6..9453617fd 100644 --- a/pkg/exchange/max/convert.go +++ b/pkg/exchange/max/convert.go @@ -333,5 +333,6 @@ func convertWebSocketOrderUpdate(u max.OrderUpdate) (*types.Order, error) { Status: toGlobalOrderStatus(u.State, executedVolume, remainingVolume), ExecutedQuantity: executedVolume, CreationTime: types.Time(time.Unix(0, u.CreatedAtMs*int64(time.Millisecond))), + UpdateTime: types.Time(time.Unix(0, u.CreatedAtMs*int64(time.Millisecond))), }, nil } From e23232c3e7aa2e7e939255b05859c9acfea6b1a4 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 18:37:34 +0800 Subject: [PATCH 30/32] max: fix timeInForce conversion --- pkg/exchange/max/convert.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/exchange/max/convert.go b/pkg/exchange/max/convert.go index 9453617fd..a43a95723 100644 --- a/pkg/exchange/max/convert.go +++ b/pkg/exchange/max/convert.go @@ -316,6 +316,11 @@ func convertWebSocketOrderUpdate(u max.OrderUpdate) (*types.Order, error) { return nil, err } + timeInForce := types.TimeInForceGTC + if u.OrderType == max.OrderTypeIOCLimit { + timeInForce = types.TimeInForceIOC + } + return &types.Order{ SubmitOrder: types.SubmitOrder{ ClientOrderID: u.ClientOID, @@ -325,7 +330,7 @@ func convertWebSocketOrderUpdate(u max.OrderUpdate) (*types.Order, error) { Quantity: fixedpoint.MustNewFromString(u.Volume), Price: fixedpoint.MustNewFromString(u.Price), StopPrice: fixedpoint.MustNewFromString(u.StopPrice), - TimeInForce: "GTC", // MAX only supports GTC + TimeInForce: timeInForce, // MAX only supports GTC GroupID: u.GroupID, }, Exchange: types.ExchangeMax, From 9b6b071d2b87357ec7d5ecf45f83461c5980185b Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 6 Mar 2022 18:47:01 +0800 Subject: [PATCH 31/32] compile and update migration package --- .../mysql/20220304153317_add_profit_table.go | 34 +++++++++++++++++++ .../20220304153309_add_profit_table.go | 34 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 pkg/migrations/mysql/20220304153317_add_profit_table.go create mode 100644 pkg/migrations/sqlite3/20220304153309_add_profit_table.go diff --git a/pkg/migrations/mysql/20220304153317_add_profit_table.go b/pkg/migrations/mysql/20220304153317_add_profit_table.go new file mode 100644 index 000000000..4cced13c0 --- /dev/null +++ b/pkg/migrations/mysql/20220304153317_add_profit_table.go @@ -0,0 +1,34 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + AddMigration(upAddProfitTable, downAddProfitTable) + +} + +func upAddProfitTable(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `profits`\n(\n `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n `strategy` VARCHAR(32) NOT NULL,\n `strategy_instance_id` VARCHAR(64) NOT NULL,\n `symbol` VARCHAR(8) NOT NULL,\n -- average_cost is the position average cost\n `average_cost` DECIMAL(16, 8) UNSIGNED NOT NULL,\n -- profit is the pnl (profit and loss)\n `profit` DECIMAL(16, 8) NOT NULL,\n -- net_profit is the pnl (profit and loss)\n `net_profit` DECIMAL(16, 8) NOT NULL,\n -- profit_margin is the pnl (profit and loss)\n `profit_margin` DECIMAL(16, 8) NOT NULL,\n -- net_profit_margin is the pnl (profit and loss)\n `net_profit_margin` DECIMAL(16, 8) NOT NULL,\n `quote_currency` VARCHAR(10) NOT NULL,\n `base_currency` VARCHAR(10) NOT NULL,\n -- -------------------------------------------------------\n -- embedded trade data --\n -- -------------------------------------------------------\n `exchange` VARCHAR(24) NOT NULL DEFAULT '',\n `is_futures` BOOLEAN NOT NULL DEFAULT FALSE,\n `is_margin` BOOLEAN NOT NULL DEFAULT FALSE,\n `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE,\n `trade_id` BIGINT UNSIGNED NOT NULL,\n -- side is the side of the trade that makes profit\n `side` VARCHAR(4) NOT NULL DEFAULT '',\n `is_buyer` BOOLEAN NOT NULL DEFAULT FALSE,\n `is_maker` BOOLEAN NOT NULL DEFAULT FALSE,\n -- price is the price of the trade that makes profit\n `price` DECIMAL(16, 8) UNSIGNED NOT NULL,\n -- quantity is the quantity of the trade that makes profit\n `quantity` DECIMAL(16, 8) UNSIGNED NOT NULL,\n -- quote_quantity is the quote quantity of the trade that makes profit\n `quote_quantity` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `traded_at` DATETIME(3) NOT NULL,\n -- fee\n `fee_in_usd` DECIMAL(16, 8) UNSIGNED,\n `fee` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `fee_currency` VARCHAR(10) NOT NULL,\n PRIMARY KEY (`gid`),\n UNIQUE KEY `trade_id` (`trade_id`)\n);") + if err != nil { + return err + } + + return err +} + +func downAddProfitTable(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `profits`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20220304153309_add_profit_table.go b/pkg/migrations/sqlite3/20220304153309_add_profit_table.go new file mode 100644 index 000000000..b95968be2 --- /dev/null +++ b/pkg/migrations/sqlite3/20220304153309_add_profit_table.go @@ -0,0 +1,34 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + AddMigration(upAddProfitTable, downAddProfitTable) + +} + +func upAddProfitTable(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `profits`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `strategy` VARCHAR(32) NOT NULL,\n `strategy_instance_id` VARCHAR(64) NOT NULL,\n `symbol` VARCHAR(8) NOT NULL,\n -- average_cost is the position average cost\n `average_cost` DECIMAL(16, 8) NOT NULL,\n -- profit is the pnl (profit and loss)\n `profit` DECIMAL(16, 8) NOT NULL,\n -- net_profit is the pnl (profit and loss)\n `net_profit` DECIMAL(16, 8) NOT NULL,\n -- profit_margin is the pnl (profit and loss)\n `profit_margin` DECIMAL(16, 8) NOT NULL,\n -- net_profit_margin is the pnl (profit and loss)\n `net_profit_margin` DECIMAL(16, 8) NOT NULL,\n `quote_currency` VARCHAR(10) NOT NULL,\n `base_currency` VARCHAR(10) NOT NULL,\n -- -------------------------------------------------------\n -- embedded trade data --\n -- -------------------------------------------------------\n `exchange` VARCHAR(24) NOT NULL DEFAULT '',\n `is_futures` BOOLEAN NOT NULL DEFAULT FALSE,\n `is_margin` BOOLEAN NOT NULL DEFAULT FALSE,\n `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE,\n `trade_id` BIGINT NOT NULL,\n -- side is the side of the trade that makes profit\n `side` VARCHAR(4) NOT NULL DEFAULT '',\n `is_buyer` BOOLEAN NOT NULL DEFAULT FALSE,\n `is_maker` BOOLEAN NOT NULL DEFAULT FALSE,\n -- price is the price of the trade that makes profit\n `price` DECIMAL(16, 8) NOT NULL,\n -- quantity is the quantity of the trade that makes profit\n `quantity` DECIMAL(16, 8) NOT NULL,\n -- trade_amount is the quote quantity of the trade that makes profit\n `quote_quantity` DECIMAL(16, 8) NOT NULL,\n `traded_at` DATETIME(3) NOT NULL,\n -- fee\n `fee_in_usd` DECIMAL(16, 8),\n `fee` DECIMAL(16, 8) NOT NULL,\n `fee_currency` VARCHAR(10) NOT NULL\n);") + if err != nil { + return err + } + + return err +} + +func downAddProfitTable(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `profits`;") + if err != nil { + return err + } + + return err +} From b8ef2eb550a1fb559b0ad64f4993dad66df6e3bb Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 7 Mar 2022 12:12:06 +0800 Subject: [PATCH 32/32] fix Test_tradeService --- pkg/service/trade_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/service/trade_test.go b/pkg/service/trade_test.go index 805bf5fd5..15ebe6938 100644 --- a/pkg/service/trade_test.go +++ b/pkg/service/trade_test.go @@ -3,12 +3,13 @@ package service import ( "context" "testing" + "time" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" - "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" ) func Test_tradeService(t *testing.T) { @@ -34,6 +35,7 @@ func Test_tradeService(t *testing.T) { Symbol: "BTCUSDT", Side: "BUY", IsBuyer: true, + Time: types.Time(time.Now()), }) assert.NoError(t, err)