From 20e61d5b5c05225065d79198294d4e51855b8f68 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 5 Dec 2021 15:26:09 +0800 Subject: [PATCH 1/6] types: extend order fields for futures --- pkg/types/order.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/types/order.go b/pkg/types/order.go index 06a5aa662..32321ccb5 100644 --- a/pkg/types/order.go +++ b/pkg/types/order.go @@ -114,6 +114,10 @@ type SubmitOrder struct { GroupID uint32 `json:"groupID,omitempty"` MarginSideEffect MarginOrderSideEffectType `json:"marginSideEffect,omitempty"` // AUTO_REPAY = repay, MARGIN_BUY = borrow, defaults to NO_SIDE_EFFECT + + // futures order fields + ReduceOnly bool `json:"reduceOnly" db:"reduce_only"` + ClosePosition bool `json:"closePosition" db:"close_position"` } func (o *SubmitOrder) String() string { From 5aa027f88392362219475c672e652312d9f66e41 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 5 Dec 2021 15:56:27 +0800 Subject: [PATCH 2/6] types: add is_futures field to the global trade --- pkg/types/trade.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/types/trade.go b/pkg/types/trade.go index 1eca4b961..f29741078 100644 --- a/pkg/types/trade.go +++ b/pkg/types/trade.go @@ -66,6 +66,7 @@ type Trade struct { FeeCurrency string `json:"feeCurrency" db:"fee_currency"` IsMargin bool `json:"isMargin" db:"is_margin"` + IsFutures bool `json:"isFutures" db:"is_futures"` IsIsolated bool `json:"isIsolated" db:"is_isolated"` StrategyID sql.NullString `json:"strategyID" db:"strategy"` From 9b56e9e32bd0f882ab14d42c29b6da0476d9e264 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 5 Dec 2021 15:59:23 +0800 Subject: [PATCH 3/6] service: add is_futures fields to order service --- pkg/service/order.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/pkg/service/order.go b/pkg/service/order.go index 04753d304..85070ad32 100644 --- a/pkg/service/order.go +++ b/pkg/service/order.go @@ -20,7 +20,9 @@ type OrderService struct { func (s *OrderService) Sync(ctx context.Context, exchange types.Exchange, symbol string, startTime time.Time) error { isMargin := false + isFutures := false isIsolated := false + if marginExchange, ok := exchange.(types.MarginExchange); ok { marginSettings := marginExchange.GetMarginSettings() isMargin = marginSettings.IsMargin @@ -30,7 +32,17 @@ func (s *OrderService) Sync(ctx context.Context, exchange types.Exchange, symbol } } - records, err := s.QueryLast(exchange.Name(), symbol, isMargin, isIsolated, 50) + if futuresExchange, ok := exchange.(types.FuturesExchange); ok { + futuresSettings := futuresExchange.GetFuturesSettings() + isFutures = futuresSettings.IsFutures + isIsolated = futuresSettings.IsIsolatedFutures + if futuresSettings.IsIsolatedFutures { + symbol = futuresSettings.IsolatedFuturesSymbol + } + } + + + records, err := s.QueryLast(exchange.Name(), symbol, isMargin, isFutures, isIsolated, 50) if err != nil { return err } @@ -78,14 +90,15 @@ func (s *OrderService) Sync(ctx context.Context, exchange types.Exchange, symbol // QueryLast queries the last order from the database -func (s *OrderService) QueryLast(ex types.ExchangeName, symbol string, isMargin, isIsolated bool, limit int) ([]types.Order, error) { - log.Infof("querying last order exchange = %s AND symbol = %s AND is_margin = %v AND is_isolated = %v", ex, symbol, isMargin, isIsolated) +func (s *OrderService) QueryLast(ex types.ExchangeName, symbol string, isMargin, isFutures, isIsolated bool, limit int) ([]types.Order, error) { + log.Infof("querying last order exchange = %s AND symbol = %s AND is_margin = %v AND is_futures = %v AND is_isolated = %v", ex, symbol, isMargin, isFutures, isIsolated) - sql := `SELECT * FROM orders WHERE exchange = :exchange AND symbol = :symbol AND is_margin = :is_margin AND is_isolated = :is_isolated ORDER BY gid DESC LIMIT :limit` + sql := `SELECT * FROM orders WHERE exchange = :exchange AND symbol = :symbol AND is_margin = :is_margin AND is_futures = :is_futures AND is_isolated = :is_isolated ORDER BY gid DESC LIMIT :limit` rows, err := s.DB.NamedQuery(sql, map[string]interface{}{ "exchange": ex, "symbol": symbol, "is_margin": isMargin, + "is_futures": isFutures, "is_isolated": isIsolated, "limit": limit, }) @@ -195,15 +208,15 @@ func (s *OrderService) scanRows(rows *sqlx.Rows) (orders []types.Order, err erro func (s *OrderService) Insert(order types.Order) (err error) { if s.DB.DriverName() == "mysql" { _, err = s.DB.NamedExec(` - INSERT INTO orders (exchange, order_id, client_order_id, order_type, status, symbol, price, stop_price, quantity, executed_quantity, side, is_working, time_in_force, created_at, updated_at, is_margin, is_isolated) - VALUES (:exchange, :order_id, :client_order_id, :order_type, :status, :symbol, :price, :stop_price, :quantity, :executed_quantity, :side, :is_working, :time_in_force, :created_at, :updated_at, :is_margin, :is_isolated) + INSERT INTO orders (exchange, order_id, client_order_id, order_type, status, symbol, price, stop_price, quantity, executed_quantity, side, is_working, time_in_force, created_at, updated_at, is_margin, is_futures, is_isolated) + VALUES (:exchange, :order_id, :client_order_id, :order_type, :status, :symbol, :price, :stop_price, :quantity, :executed_quantity, :side, :is_working, :time_in_force, :created_at, :updated_at, :is_margin, :is_futures, :is_isolated) ON DUPLICATE KEY UPDATE status=:status, executed_quantity=:executed_quantity, is_working=:is_working, updated_at=:updated_at`, order) return err } _, err = s.DB.NamedExec(` - INSERT INTO orders (exchange, order_id, client_order_id, order_type, status, symbol, price, stop_price, quantity, executed_quantity, side, is_working, time_in_force, created_at, updated_at, is_margin, is_isolated) - VALUES (:exchange, :order_id, :client_order_id, :order_type, :status, :symbol, :price, :stop_price, :quantity, :executed_quantity, :side, :is_working, :time_in_force, :created_at, :updated_at, :is_margin, :is_isolated) + INSERT INTO orders (exchange, order_id, client_order_id, order_type, status, symbol, price, stop_price, quantity, executed_quantity, side, is_working, time_in_force, created_at, updated_at, is_margin, is_futures, is_isolated) + VALUES (:exchange, :order_id, :client_order_id, :order_type, :status, :symbol, :price, :stop_price, :quantity, :executed_quantity, :side, :is_working, :time_in_force, :created_at, :updated_at, :is_margin, :is_futures, :is_isolated) `, order) return err From 874c7b39faa1b17b240ad2a0030f9cbaea4c3744 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 5 Dec 2021 15:59:42 +0800 Subject: [PATCH 4/6] service: add is_futures fields to trade service --- pkg/service/trade.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/pkg/service/trade.go b/pkg/service/trade.go index f089b8e15..45ad9049c 100644 --- a/pkg/service/trade.go +++ b/pkg/service/trade.go @@ -52,7 +52,9 @@ func NewTradeService(db *sqlx.DB) *TradeService { func (s *TradeService) Sync(ctx context.Context, exchange types.Exchange, symbol string) error { isMargin := false + isFutures := false isIsolated := false + if marginExchange, ok := exchange.(types.MarginExchange); ok { marginSettings := marginExchange.GetMarginSettings() isMargin = marginSettings.IsMargin @@ -62,8 +64,18 @@ func (s *TradeService) Sync(ctx context.Context, exchange types.Exchange, symbol } } + if futuresExchange, ok := exchange.(types.FuturesExchange); ok { + futuresSettings := futuresExchange.GetFuturesSettings() + isFutures = futuresSettings.IsFutures + isIsolated = futuresSettings.IsIsolatedFutures + if futuresSettings.IsIsolatedFutures { + symbol = futuresSettings.IsolatedFuturesSymbol + } + } + + // records descending ordered - records, err := s.QueryLast(exchange.Name(), symbol, isMargin, isIsolated, 50) + records, err := s.QueryLast(exchange.Name(), symbol, isMargin, isFutures, isIsolated, 50) if err != nil { return err } @@ -265,14 +277,15 @@ func generateMysqlTradingVolumeQuerySQL(options TradingVolumeQueryOptions) strin } // QueryLast queries the last trade from the database -func (s *TradeService) QueryLast(ex types.ExchangeName, symbol string, isMargin, isIsolated bool, limit int) ([]types.Trade, error) { - log.Debugf("querying last trade exchange = %s AND symbol = %s AND is_margin = %v AND is_isolated = %v", ex, symbol, isMargin, isIsolated) +func (s *TradeService) QueryLast(ex types.ExchangeName, symbol string, isMargin, isFutures, isIsolated bool, limit int) ([]types.Trade, error) { + log.Debugf("querying last trade exchange = %s AND symbol = %s AND is_margin = %v AND is_futures = %v AND is_isolated = %v", ex, symbol, isMargin, isFutures, isIsolated) - sql := "SELECT * FROM trades WHERE exchange = :exchange AND symbol = :symbol AND is_margin = :is_margin AND is_isolated = :is_isolated ORDER BY gid DESC LIMIT :limit" + sql := "SELECT * FROM trades WHERE exchange = :exchange AND symbol = :symbol AND is_margin = :is_margin AND is_futures = :is_futures AND is_isolated = :is_isolated ORDER BY gid DESC LIMIT :limit" rows, err := s.DB.NamedQuery(sql, map[string]interface{}{ "symbol": symbol, "exchange": ex, "is_margin": isMargin, + "is_futures": isFutures, "is_isolated": isIsolated, "limit": limit, }) @@ -439,8 +452,8 @@ func (s *TradeService) scanRows(rows *sqlx.Rows) (trades []types.Trade, err erro func (s *TradeService) Insert(trade types.Trade) error { _, err := s.DB.NamedExec(` - INSERT INTO trades (id, exchange, order_id, symbol, price, quantity, quote_quantity, side, is_buyer, is_maker, fee, fee_currency, traded_at, is_margin, is_isolated) - VALUES (:id, :exchange, :order_id, :symbol, :price, :quantity, :quote_quantity, :side, :is_buyer, :is_maker, :fee, :fee_currency, :traded_at, :is_margin, :is_isolated)`, + INSERT INTO trades (id, exchange, order_id, symbol, price, quantity, quote_quantity, side, is_buyer, is_maker, fee, fee_currency, traded_at, is_margin, is_futures, is_isolated) + VALUES (:id, :exchange, :order_id, :symbol, :price, :quantity, :quote_quantity, :side, :is_buyer, :is_maker, :fee, :fee_currency, :traded_at, :is_margin, :is_futures, :is_isolated)`, trade) return err } From 5cc9e74823710aa6c02f72c414cd7cb6c15e2860 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 5 Dec 2021 16:23:54 +0800 Subject: [PATCH 5/6] add is_futures column migration sql --- .../20211205162043_add_is_futures_column.sql | 18 ++++++++++++++++++ .../20211205162302_add_is_futures_column.sql | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 migrations/mysql/20211205162043_add_is_futures_column.sql create mode 100644 migrations/sqlite3/20211205162302_add_is_futures_column.sql diff --git a/migrations/mysql/20211205162043_add_is_futures_column.sql b/migrations/mysql/20211205162043_add_is_futures_column.sql new file mode 100644 index 000000000..b2929edd2 --- /dev/null +++ b/migrations/mysql/20211205162043_add_is_futures_column.sql @@ -0,0 +1,18 @@ +-- +up +-- +begin +ALTER TABLE `trades` ADD COLUMN `is_futures` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end + +-- +begin +ALTER TABLE `orders` ADD COLUMN `is_futures` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end + +-- +down + +-- +begin +ALTER TABLE `trades` DROP COLUMN `is_futures`; +-- +end + +-- +begin +ALTER TABLE `orders` DROP COLUMN `is_futures`; +-- +end diff --git a/migrations/sqlite3/20211205162302_add_is_futures_column.sql b/migrations/sqlite3/20211205162302_add_is_futures_column.sql new file mode 100644 index 000000000..ca020183e --- /dev/null +++ b/migrations/sqlite3/20211205162302_add_is_futures_column.sql @@ -0,0 +1,18 @@ +-- +up +-- +begin +ALTER TABLE `trades` ADD COLUMN `is_futures` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end + +-- +begin +ALTER TABLE `orders` ADD COLUMN `is_futures` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end + +-- +down + +-- +begin +ALTER TABLE `trades` RENAME COLUMN `is_futures` TO `is_futures_deleted`; +-- +end + +-- +begin +ALTER TABLE `orders` RENAME COLUMN `is_futures` TO `is_futures_deleted`; +-- +end From 977b2a685278cfd806d7dab4bb9f01d3174a13bc Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 8 Dec 2021 19:51:22 +0800 Subject: [PATCH 6/6] add rockhopper usage --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 490d5793c..52a079702 100644 --- a/README.md +++ b/README.md @@ -535,7 +535,7 @@ rockhopper --config rockhopper_sqlite.yaml create --type sql add_pnl_column rockhopper --config rockhopper_mysql.yaml create --type sql add_pnl_column ``` -or +or you can use the util script: ``` bash utils/generate-new-migration.sh add_pnl_column @@ -558,6 +558,21 @@ Then run the following command to compile the migration files into go files: make migrations ``` + +If you want to override the DSN and the Driver defined in the YAML config file, you can add some env vars in your dotenv file like this: + +```shell +ROCKHOPPER_DRIVER=mysql +ROCKHOPPER_DIALECT=mysql +ROCKHOPPER_DSN="root:123123@unix(/opt/local/var/run/mysql57/mysqld.sock)/bbgo" +``` + +And then, run: + +```shell +dotenv -f .env.local -- rockhopper --config rockhopper_mysql.yaml up +``` + ### Setup frontend development environment ```sh