From 893b513605f880c264a7762e3b92ef2cc181d0db Mon Sep 17 00:00:00 2001 From: Jui-Nan Lin Date: Fri, 5 Feb 2021 10:12:10 +0800 Subject: [PATCH 01/33] feat: add limit option for pnl command --- pkg/cmd/pnl.go | 7 +++++++ pkg/service/trade.go | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/cmd/pnl.go b/pkg/cmd/pnl.go index 9e27c6f5b..9034890bf 100644 --- a/pkg/cmd/pnl.go +++ b/pkg/cmd/pnl.go @@ -21,6 +21,7 @@ import ( func init() { PnLCmd.Flags().String("exchange", "", "target exchange") PnLCmd.Flags().String("symbol", "BTCUSDT", "trading symbol") + PnLCmd.Flags().Int("limit", 500, "number of orders") RootCmd.AddCommand(PnLCmd) } @@ -46,6 +47,11 @@ var PnLCmd = &cobra.Command{ return err } + limit, err := cmd.Flags().GetInt("limit") + if err != nil { + return err + } + exchange, err := cmdutil.NewExchange(exchangeName) if err != nil { return err @@ -67,6 +73,7 @@ var PnLCmd = &cobra.Command{ trades, err = tradeService.Query(service.QueryTradesOptions{ Exchange: exchange.Name(), Symbol: symbol, + Limit: limit, }) } diff --git a/pkg/service/trade.go b/pkg/service/trade.go index 6739fbf07..859d53430 100644 --- a/pkg/service/trade.go +++ b/pkg/service/trade.go @@ -163,13 +163,13 @@ func (s *TradeService) QueryForTradingFeeCurrency(ex types.ExchangeName, symbol return s.scanRows(rows) } -// Only return 500 items. type QueryTradesOptions struct { Exchange types.ExchangeName Symbol string LastGID int64 // ASC or DESC Ordering string + Limit int } func (s *TradeService) Query(options QueryTradesOptions) ([]types.Trade, error) { @@ -225,7 +225,7 @@ func queryTradesSQL(options QueryTradesOptions) string { sql += ` ORDER BY gid ` + ordering - sql += ` LIMIT ` + strconv.Itoa(500) + sql += ` LIMIT ` + strconv.Itoa(options.Limit) return sql } From 8eb8fb105d52c446633a9e3fd75f84cea6ddf11c Mon Sep 17 00:00:00 2001 From: Jui-Nan Lin Date: Fri, 5 Feb 2021 12:42:59 +0800 Subject: [PATCH 02/33] test: fix query trading test --- pkg/service/trade_test.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/service/trade_test.go b/pkg/service/trade_test.go index 35e4e486d..5e0381b93 100644 --- a/pkg/service/trade_test.go +++ b/pkg/service/trade_test.go @@ -27,23 +27,23 @@ func Test_queryTradingVolumeSQL(t *testing.T) { func Test_queryTradesSQL(t *testing.T) { t.Run("generate order by clause by Ordering option", func(t *testing.T) { - assert.Equal(t, "SELECT * FROM trades ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{})) - assert.Equal(t, "SELECT * FROM trades ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{Ordering: "ASC"})) - assert.Equal(t, "SELECT * FROM trades ORDER BY gid DESC LIMIT 500", queryTradesSQL(QueryTradesOptions{Ordering: "DESC"})) + assert.Equal(t, "SELECT * FROM trades ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{Limit: 500})) + assert.Equal(t, "SELECT * FROM trades ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{Ordering: "ASC", Limit: 500})) + assert.Equal(t, "SELECT * FROM trades ORDER BY gid DESC LIMIT 500", queryTradesSQL(QueryTradesOptions{Ordering: "DESC", Limit: 500})) }) t.Run("filter by exchange name", func(t *testing.T) { - assert.Equal(t, "SELECT * FROM trades WHERE exchange = :exchange ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{Exchange: "max"})) + assert.Equal(t, "SELECT * FROM trades WHERE exchange = :exchange ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{Exchange: "max", Limit: 500})) }) t.Run("filter by symbol", func(t *testing.T) { - assert.Equal(t, "SELECT * FROM trades WHERE symbol = :symbol ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{Symbol: "eth"})) + assert.Equal(t, "SELECT * FROM trades WHERE symbol = :symbol ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{Symbol: "eth", Limit: 500})) }) t.Run("GID ordering", func(t *testing.T) { - assert.Equal(t, "SELECT * FROM trades WHERE gid > :gid ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{LastGID: 1})) - assert.Equal(t, "SELECT * FROM trades WHERE gid > :gid ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{LastGID: 1, Ordering: "ASC"})) - assert.Equal(t, "SELECT * FROM trades WHERE gid < :gid ORDER BY gid DESC LIMIT 500", queryTradesSQL(QueryTradesOptions{LastGID: 1, Ordering: "DESC"})) + assert.Equal(t, "SELECT * FROM trades WHERE gid > :gid ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{LastGID: 1, Limit: 500})) + assert.Equal(t, "SELECT * FROM trades WHERE gid > :gid ORDER BY gid ASC LIMIT 500", queryTradesSQL(QueryTradesOptions{LastGID: 1, Ordering: "ASC", Limit: 500})) + assert.Equal(t, "SELECT * FROM trades WHERE gid < :gid ORDER BY gid DESC LIMIT 500", queryTradesSQL(QueryTradesOptions{LastGID: 1, Ordering: "DESC", Limit: 500})) }) t.Run("convert all options", func(t *testing.T) { @@ -52,6 +52,7 @@ func Test_queryTradesSQL(t *testing.T) { Symbol: "btc", LastGID: 123, Ordering: "DESC", + Limit: 500, })) }) } From ea0c20cfe7d240dc9d55c8eabfee3fef2cb6ff54 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 13:04:52 +0800 Subject: [PATCH 03/33] rename enableApiServer to enableWebServer --- pkg/cmd/run.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index ccb0e7415..b246d2d59 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -331,7 +331,7 @@ func run(cmd *cobra.Command, args []string) error { return err } - enableApiServer, err := cmd.Flags().GetBool("enable-web-server") + enableWebServer, err := cmd.Flags().GetBool("enable-web-server") if err != nil { return err } @@ -348,10 +348,7 @@ func run(cmd *cobra.Command, args []string) error { var userConfig = &bbgo.Config{} - if setup { - log.Infof("running in setup mode, skip reading config file") - enableApiServer = true - } else { + if !setup { // if it's not setup, then the config file option is required. if len(configFile) == 0 { return errors.New("--config option is required") @@ -377,7 +374,7 @@ func run(cmd *cobra.Command, args []string) error { } if setup { - return runSetup(ctx, userConfig, enableApiServer) + return runSetup(ctx, userConfig, true) } userConfig, err = bbgo.Load(configFile, true) @@ -385,7 +382,7 @@ func run(cmd *cobra.Command, args []string) error { return err } - return runConfig(ctx, userConfig, enableApiServer) + return runConfig(ctx, userConfig, enableWebServer) } return runWrapperBinary(ctx, userConfig, cmd, args) From 8cb77075d03e7e9bb4a16f00f650f44e6898dfe4 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 13:26:41 +0800 Subject: [PATCH 04/33] .travis.yml: get rockhopper before we test --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c36fe7efb..ac48e49fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ go: services: - redis-server before_script: +- go get github.com/c9s/rockhopper - go mod download script: - go test -v ./pkg/... From 007b67976829a53500bd97567eee167d542bfdb5 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 14:15:28 +0800 Subject: [PATCH 05/33] add sqlite3 migrations --- migrations/20200721225616_trades.sql | 3 +- migrations/sqlite3/20200721225616_trades.sql | 20 +++++++++ .../sqlite3/20200819054742_trade_index.sql | 9 ++++ migrations/sqlite3/20201102222546_orders.sql | 25 +++++++++++ .../20201103173342_trades_add_order_id.sql | 5 +++ .../20201105092857_trades_index_fix.sql | 19 ++++++++ .../20201105093056_orders_add_index.sql | 7 +++ migrations/sqlite3/20201106114742_klines.sql | 44 +++++++++++++++++++ .../20201211175751_fix_symbol_length.sql | 5 +++ .../20210118163847_fix_unique_index.sql | 9 ++++ .../20210119232826_add_margin_columns.sql | 33 ++++++++++++++ ...10129182704_trade_price_quantity_index.sql | 10 +++++ 12 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 migrations/sqlite3/20200721225616_trades.sql create mode 100644 migrations/sqlite3/20200819054742_trade_index.sql create mode 100644 migrations/sqlite3/20201102222546_orders.sql create mode 100644 migrations/sqlite3/20201103173342_trades_add_order_id.sql create mode 100644 migrations/sqlite3/20201105092857_trades_index_fix.sql create mode 100644 migrations/sqlite3/20201105093056_orders_add_index.sql create mode 100644 migrations/sqlite3/20201106114742_klines.sql create mode 100644 migrations/sqlite3/20201211175751_fix_symbol_length.sql create mode 100644 migrations/sqlite3/20210118163847_fix_unique_index.sql create mode 100644 migrations/sqlite3/20210119232826_add_margin_columns.sql create mode 100644 migrations/sqlite3/20210129182704_trade_price_quantity_index.sql diff --git a/migrations/20200721225616_trades.sql b/migrations/20200721225616_trades.sql index 12d4e4c77..27760ac23 100644 --- a/migrations/20200721225616_trades.sql +++ b/migrations/20200721225616_trades.sql @@ -19,5 +19,6 @@ CREATE TABLE `trades` PRIMARY KEY (`gid`), UNIQUE KEY `id` (`id`) ); + -- +down -DROP TABLE `trades`; +DROP TABLE IF EXISTS `trades`; diff --git a/migrations/sqlite3/20200721225616_trades.sql b/migrations/sqlite3/20200721225616_trades.sql new file mode 100644 index 000000000..fcd5f8f50 --- /dev/null +++ b/migrations/sqlite3/20200721225616_trades.sql @@ -0,0 +1,20 @@ +-- +up +CREATE TABLE `trades` +( + `gid` INTEGER PRIMARY KEY AUTOINCREMENT, + `id` INTEGER, + `exchange` TEXT NOT NULL DEFAULT '', + `symbol` TEXT NOT NULL, + `price` DECIMAL(16, 8) NOT NULL, + `quantity` DECIMAL(16, 8) NOT NULL, + `quote_quantity` DECIMAL(16, 8) NOT NULL, + `fee` DECIMAL(16, 8) NOT NULL, + `fee_currency` VARCHAR(4) NOT NULL, + `is_buyer` BOOLEAN NOT NULL DEFAULT FALSE, + `is_maker` BOOLEAN NOT NULL DEFAULT FALSE, + `side` VARCHAR(4) NOT NULL DEFAULT '', + `traded_at` DATETIME(3) NOT NULL +); + +-- +down +DROP TABLE IF EXISTS `trades`; diff --git a/migrations/sqlite3/20200819054742_trade_index.sql b/migrations/sqlite3/20200819054742_trade_index.sql new file mode 100644 index 000000000..6ce90b1bd --- /dev/null +++ b/migrations/sqlite3/20200819054742_trade_index.sql @@ -0,0 +1,9 @@ +-- +up +CREATE INDEX trades_symbol ON trades(symbol); +CREATE INDEX trades_symbol_fee_currency ON trades(symbol, fee_currency, traded_at); +CREATE INDEX trades_traded_at_symbol ON trades(traded_at, symbol); + +-- +down +DROP INDEX trades_symbol ON trades; +DROP INDEX trades_symbol_fee_currency ON trades; +DROP INDEX trades_traded_at_symbol ON trades; diff --git a/migrations/sqlite3/20201102222546_orders.sql b/migrations/sqlite3/20201102222546_orders.sql new file mode 100644 index 000000000..561afd06a --- /dev/null +++ b/migrations/sqlite3/20201102222546_orders.sql @@ -0,0 +1,25 @@ +-- +up +CREATE TABLE `orders` +( + `gid` INTEGER PRIMARY KEY AUTOINCREMENT, + + `exchange` VARCHAR NOT NULL DEFAULT '', + -- order_id is the order id returned from the exchange + `order_id` INTEGER NOT NULL, + `client_order_id` VARCHAR NOT NULL DEFAULT '', + `order_type` VARCHAR NOT NULL, + `symbol` VARCHAR NOT NULL, + `status` VARCHAR NOT NULL, + `time_in_force` VARCHAR NOT NULL, + `price` DECIMAL(16, 8) NOT NULL, + `stop_price` DECIMAL(16, 8) NOT NULL, + `quantity` DECIMAL(16, 8) NOT NULL, + `executed_quantity` DECIMAL(16, 8) NOT NULL DEFAULT 0.0, + `side` VARCHAR NOT NULL DEFAULT '', + `is_working` BOOLEAN NOT NULL DEFAULT FALSE, + `created_at` DATETIME(3) NOT NULL, + `updated_at` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- +down +DROP TABLE IF EXISTS `orders`; diff --git a/migrations/sqlite3/20201103173342_trades_add_order_id.sql b/migrations/sqlite3/20201103173342_trades_add_order_id.sql new file mode 100644 index 000000000..7ff284200 --- /dev/null +++ b/migrations/sqlite3/20201103173342_trades_add_order_id.sql @@ -0,0 +1,5 @@ +-- +up +ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER NOT NULL; + +-- +down +ALTER TABLE `trades` RENAME COLUMN `order_id` TO `order_id_deleted`; diff --git a/migrations/sqlite3/20201105092857_trades_index_fix.sql b/migrations/sqlite3/20201105092857_trades_index_fix.sql new file mode 100644 index 000000000..44f58e768 --- /dev/null +++ b/migrations/sqlite3/20201105092857_trades_index_fix.sql @@ -0,0 +1,19 @@ +-- +up +DROP INDEX IF EXISTS trades_symbol; +DROP INDEX IF EXISTS trades_symbol_fee_currency; +DROP INDEX IF EXISTS trades_traded_at_symbol; + +CREATE INDEX trades_symbol ON trades (exchange, symbol); +CREATE INDEX trades_symbol_fee_currency ON trades (exchange, symbol, fee_currency, traded_at); +CREATE INDEX trades_traded_at_symbol ON trades (exchange, traded_at, symbol); + +-- +down +DROP INDEX IF EXISTS trades_symbol ON trades; +DROP INDEX IF EXISTS trades_symbol_fee_currency ON trades; +DROP INDEX IF EXISTS trades_traded_at_symbol ON trades; + +CREATE INDEX trades_symbol ON trades (symbol); +CREATE INDEX trades_symbol_fee_currency ON trades (symbol, fee_currency, traded_at); +CREATE INDEX trades_traded_at_symbol ON trades (traded_at, symbol); + + diff --git a/migrations/sqlite3/20201105093056_orders_add_index.sql b/migrations/sqlite3/20201105093056_orders_add_index.sql new file mode 100644 index 000000000..99834a551 --- /dev/null +++ b/migrations/sqlite3/20201105093056_orders_add_index.sql @@ -0,0 +1,7 @@ +-- +up +CREATE INDEX orders_symbol ON orders (exchange, symbol); +CREATE UNIQUE INDEX orders_order_id ON orders (order_id, exchange); + +-- +down +DROP INDEX IF EXISTS orders_symbol; +DROP INDEX IF EXISTS orders_order_id; diff --git a/migrations/sqlite3/20201106114742_klines.sql b/migrations/sqlite3/20201106114742_klines.sql new file mode 100644 index 000000000..72c679206 --- /dev/null +++ b/migrations/sqlite3/20201106114742_klines.sql @@ -0,0 +1,44 @@ +-- +up +-- +begin +CREATE TABLE `klines` +( + `gid` INTEGER PRIMARY KEY AUTOINCREMENT, + `exchange` VARCHAR(10) NOT NULL, + `start_time` DATETIME(3) NOT NULL, + `end_time` DATETIME(3) NOT NULL, + `interval` VARCHAR(3) NOT NULL, + `symbol` VARCHAR(7) NOT NULL, + `open` DECIMAL(16, 8) NOT NULL, + `high` DECIMAL(16, 8) NOT NULL, + `low` DECIMAL(16, 8) NOT NULL, + `close` DECIMAL(16, 8) NOT NULL DEFAULT 0.0, + `volume` DECIMAL(16, 8) NOT NULL DEFAULT 0.0, + `closed` BOOLEAN NOT NULL DEFAULT TRUE, + `last_trade_id` INT NOT NULL DEFAULT 0, + `num_trades` INT NOT NULL DEFAULT 0 +); +-- +end + +-- +begin +CREATE INDEX `klines_end_time_symbol_interval` ON klines (`end_time`, `symbol`, `interval`); +-- +end + +-- +begin +CREATE TABLE `okex_klines` AS SELECT * FROM `klines` WHERE 0 +-- +end + +-- +begin +CREATE TABLE `binance_klines` AS SELECT * FROM `klines` WHERE 0 +-- +end + +-- +begin +CREATE TABLE `max_klines` AS SELECT * FROM `klines` WHERE 0 +-- +end + +-- +down +DROP INDEX IF EXISTS `klines_end_time_symbol_interval`; +DROP TABLE IF EXISTS `binance_klines`; +DROP TABLE IF EXISTS `okex_klines`; +DROP TABLE IF EXISTS `max_klines`; +DROP TABLE IF EXISTS `klines`; + diff --git a/migrations/sqlite3/20201211175751_fix_symbol_length.sql b/migrations/sqlite3/20201211175751_fix_symbol_length.sql new file mode 100644 index 000000000..06569c667 --- /dev/null +++ b/migrations/sqlite3/20201211175751_fix_symbol_length.sql @@ -0,0 +1,5 @@ +-- +up +SELECT 1; + +-- +down +SELECT 1; diff --git a/migrations/sqlite3/20210118163847_fix_unique_index.sql b/migrations/sqlite3/20210118163847_fix_unique_index.sql new file mode 100644 index 000000000..60ada793e --- /dev/null +++ b/migrations/sqlite3/20210118163847_fix_unique_index.sql @@ -0,0 +1,9 @@ +-- +up +-- +begin +CREATE UNIQUE INDEX `trade_unique_id` ON `trades` (`exchange`,`symbol`, `side`, `id`); +-- +end + +-- +down +-- +begin +DROP INDEX IF EXISTS `trade_unique_id`; +-- +end diff --git a/migrations/sqlite3/20210119232826_add_margin_columns.sql b/migrations/sqlite3/20210119232826_add_margin_columns.sql new file mode 100644 index 000000000..eaff7b613 --- /dev/null +++ b/migrations/sqlite3/20210119232826_add_margin_columns.sql @@ -0,0 +1,33 @@ +-- +up +-- +begin +ALTER TABLE `trades` ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end +-- +begin +ALTER TABLE `trades` ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end + +-- +begin +ALTER TABLE `orders` ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end + +-- +begin +ALTER TABLE `orders` ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end + +-- +down + +-- +begin +ALTER TABLE `trades` RENAME COLUMN `is_margin` TO `is_margin_deleted`; +-- +end + +-- +begin +ALTER TABLE `trades` RENAME COLUMN `is_isolated` TO `is_isolated_deleted`; +-- +end + +-- +begin +ALTER TABLE `orders` RENAME COLUMN `is_margin` TO `is_margin_deleted`; +-- +end + +-- +begin +ALTER TABLE `orders` RENAME COLUMN `is_isolated` TO `is_isolated_deleted`; +-- +end diff --git a/migrations/sqlite3/20210129182704_trade_price_quantity_index.sql b/migrations/sqlite3/20210129182704_trade_price_quantity_index.sql new file mode 100644 index 000000000..196f7467d --- /dev/null +++ b/migrations/sqlite3/20210129182704_trade_price_quantity_index.sql @@ -0,0 +1,10 @@ +-- +up +-- +begin +CREATE INDEX trades_price_quantity ON trades (order_id,price,quantity); +-- +end + +-- +down + +-- +begin +DROP INDEX trades_price_quantity; +-- +end From 16b7944b925725fe838f9c77d4f0449cff8eb553 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 14:15:44 +0800 Subject: [PATCH 06/33] add rockhopper config for sqlite3 --- rockhopper_sqlite.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 rockhopper_sqlite.yaml diff --git a/rockhopper_sqlite.yaml b/rockhopper_sqlite.yaml new file mode 100644 index 000000000..259e7fe67 --- /dev/null +++ b/rockhopper_sqlite.yaml @@ -0,0 +1,5 @@ +--- +driver: sqlite3 +dialect: sqlite3 +dsn: "bbgo.sqlite3" +migrationsDir: migrations/sqlite3 From ccb1708fd96fa813ba96a97e62818918fde1068b Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 14:20:07 +0800 Subject: [PATCH 07/33] add script for testing sqlite3 migration --- .travis.yml | 1 + migrations/sqlite3/20200819054742_trade_index.sql | 6 +++--- migrations/sqlite3/20201105092857_trades_index_fix.sql | 6 +++--- scripts/test-sqlite3-migrations.sh | 2 ++ 4 files changed, 9 insertions(+), 6 deletions(-) create mode 100755 scripts/test-sqlite3-migrations.sh diff --git a/.travis.yml b/.travis.yml index ac48e49fe..595d79af6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,4 +9,5 @@ before_script: - go get github.com/c9s/rockhopper - go mod download script: +- bash scripts/test-sqlite3-migrations.sh - go test -v ./pkg/... diff --git a/migrations/sqlite3/20200819054742_trade_index.sql b/migrations/sqlite3/20200819054742_trade_index.sql index 6ce90b1bd..e6dba452b 100644 --- a/migrations/sqlite3/20200819054742_trade_index.sql +++ b/migrations/sqlite3/20200819054742_trade_index.sql @@ -4,6 +4,6 @@ CREATE INDEX trades_symbol_fee_currency ON trades(symbol, fee_currency, traded_a CREATE INDEX trades_traded_at_symbol ON trades(traded_at, symbol); -- +down -DROP INDEX trades_symbol ON trades; -DROP INDEX trades_symbol_fee_currency ON trades; -DROP INDEX trades_traded_at_symbol ON trades; +DROP INDEX trades_symbol; +DROP INDEX trades_symbol_fee_currency; +DROP INDEX trades_traded_at_symbol; diff --git a/migrations/sqlite3/20201105092857_trades_index_fix.sql b/migrations/sqlite3/20201105092857_trades_index_fix.sql index 44f58e768..30a955406 100644 --- a/migrations/sqlite3/20201105092857_trades_index_fix.sql +++ b/migrations/sqlite3/20201105092857_trades_index_fix.sql @@ -8,9 +8,9 @@ CREATE INDEX trades_symbol_fee_currency ON trades (exchange, symbol, fee_currenc CREATE INDEX trades_traded_at_symbol ON trades (exchange, traded_at, symbol); -- +down -DROP INDEX IF EXISTS trades_symbol ON trades; -DROP INDEX IF EXISTS trades_symbol_fee_currency ON trades; -DROP INDEX IF EXISTS trades_traded_at_symbol ON trades; +DROP INDEX IF EXISTS trades_symbol; +DROP INDEX IF EXISTS trades_symbol_fee_currency; +DROP INDEX IF EXISTS trades_traded_at_symbol; CREATE INDEX trades_symbol ON trades (symbol); CREATE INDEX trades_symbol_fee_currency ON trades (symbol, fee_currency, traded_at); diff --git a/scripts/test-sqlite3-migrations.sh b/scripts/test-sqlite3-migrations.sh new file mode 100755 index 000000000..83c5b683c --- /dev/null +++ b/scripts/test-sqlite3-migrations.sh @@ -0,0 +1,2 @@ +#!/bin/bash +rm -v bbgo.sqlite3 && rockhopper --config rockhopper_sqlite.yaml up && rockhopper --config rockhopper_sqlite.yaml down --to 1 From 01ea0885dcffbd0a60fedf25ec1620fb9c04f1c8 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 14:33:12 +0800 Subject: [PATCH 08/33] scripts: use force flag to remove bbgo.sqlite3 --- scripts/test-sqlite3-migrations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-sqlite3-migrations.sh b/scripts/test-sqlite3-migrations.sh index 83c5b683c..c552aed86 100755 --- a/scripts/test-sqlite3-migrations.sh +++ b/scripts/test-sqlite3-migrations.sh @@ -1,2 +1,2 @@ #!/bin/bash -rm -v bbgo.sqlite3 && rockhopper --config rockhopper_sqlite.yaml up && rockhopper --config rockhopper_sqlite.yaml down --to 1 +rm -fv bbgo.sqlite3 && rockhopper --config rockhopper_sqlite.yaml up && rockhopper --config rockhopper_sqlite.yaml down --to 1 From b6da7ee2f26a4d51f962d43333c9b8269fd06076 Mon Sep 17 00:00:00 2001 From: Jui-Nan Lin Date: Fri, 5 Feb 2021 14:49:42 +0800 Subject: [PATCH 09/33] fix(pnl): should be trades --- pkg/cmd/pnl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cmd/pnl.go b/pkg/cmd/pnl.go index 9034890bf..2ae496a39 100644 --- a/pkg/cmd/pnl.go +++ b/pkg/cmd/pnl.go @@ -21,7 +21,7 @@ import ( func init() { PnLCmd.Flags().String("exchange", "", "target exchange") PnLCmd.Flags().String("symbol", "BTCUSDT", "trading symbol") - PnLCmd.Flags().Int("limit", 500, "number of orders") + PnLCmd.Flags().Int("limit", 500, "number of trades") RootCmd.AddCommand(PnLCmd) } From f44d6a323ad4c4878b4778c23b2705ab9d5b3f25 Mon Sep 17 00:00:00 2001 From: ycdesu Date: Fri, 5 Feb 2021 22:30:47 +0800 Subject: [PATCH 10/33] http: move response helper to util --- pkg/exchange/max/maxapi/restapi.go | 45 ++++++------------------------ pkg/util/http_response.go | 38 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 37 deletions(-) create mode 100644 pkg/util/http_response.go diff --git a/pkg/exchange/max/maxapi/restapi.go b/pkg/exchange/max/maxapi/restapi.go index 27d14e3f8..1a0e9c3d4 100644 --- a/pkg/exchange/max/maxapi/restapi.go +++ b/pkg/exchange/max/maxapi/restapi.go @@ -20,6 +20,8 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + + "github.com/c9s/bbgo/pkg/util" ) const ( @@ -46,37 +48,6 @@ var serverTimestamp = time.Now().Unix() // reqCount is used for nonce, this variable counts the API request count. var reqCount int64 = 0 -// Response is wrapper for standard http.Response and provides -// more methods. -type Response struct { - *http.Response - - // Body overrides the composited Body field. - Body []byte -} - -// newResponse is a wrapper of the http.Response instance, it reads the response body and close the file. -func newResponse(r *http.Response) (response *Response, err error) { - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, err - } - - err = r.Body.Close() - response = &Response{Response: r, Body: body} - return response, err -} - -// String converts response body to string. -// An empty string will be returned if error. -func (r *Response) String() string { - return string(r.Body) -} - -func (r *Response) DecodeJSON(o interface{}) error { - return json.Unmarshal(r.Body, o) -} - type RestClient struct { client *http.Client @@ -290,14 +261,14 @@ func (c *RestClient) Do(req *http.Request) (resp *http.Response, err error) { } // sendRequest sends the request to the API server and handle the response -func (c *RestClient) sendRequest(req *http.Request) (*Response, error) { +func (c *RestClient) sendRequest(req *http.Request) (*util.Response, error) { resp, err := c.Do(req) if err != nil { return nil, err } // newResponse reads the response body and return a new Response object - response, err := newResponse(resp) + response, err := util.NewResponse(resp) if err != nil { return response, err } @@ -314,7 +285,7 @@ func (c *RestClient) sendRequest(req *http.Request) (*Response, error) { return response, nil } -func (c *RestClient) sendAuthenticatedRequest(m string, refURL string, data map[string]interface{}) (*Response, error) { +func (c *RestClient) sendAuthenticatedRequest(m string, refURL string, data map[string]interface{}) (*util.Response, error) { req, err := c.newAuthenticatedRequest(m, refURL, data) if err != nil { return nil, err @@ -374,7 +345,7 @@ type ErrorField struct { } type ErrorResponse struct { - *Response + *util.Response Err ErrorField `json:"error"` } @@ -389,13 +360,13 @@ func (r *ErrorResponse) Error() string { } // isError check the response status code so see if a response is an error. -func isError(response *Response) bool { +func isError(response *util.Response) bool { var c = response.StatusCode return c < 200 || c > 299 } // toErrorResponse tries to convert/parse the server response to the standard Error interface object -func toErrorResponse(response *Response) (errorResponse *ErrorResponse, err error) { +func toErrorResponse(response *util.Response) (errorResponse *ErrorResponse, err error) { errorResponse = &ErrorResponse{Response: response} contentType := response.Header.Get("content-type") diff --git a/pkg/util/http_response.go b/pkg/util/http_response.go new file mode 100644 index 000000000..05698ec3c --- /dev/null +++ b/pkg/util/http_response.go @@ -0,0 +1,38 @@ +package util + +import ( + "encoding/json" + "io/ioutil" + "net/http" +) + +// Response is wrapper for standard http.Response and provides +// more methods. +type Response struct { + *http.Response + + // Body overrides the composited Body field. + Body []byte +} + +// newResponse is a wrapper of the http.Response instance, it reads the response body and close the file. +func NewResponse(r *http.Response) (response *Response, err error) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, err + } + + err = r.Body.Close() + response = &Response{Response: r, Body: body} + return response, err +} + +// String converts response body to string. +// An empty string will be returned if error. +func (r *Response) String() string { + return string(r.Body) +} + +func (r *Response) DecodeJSON(o interface{}) error { + return json.Unmarshal(r.Body, o) +} From 06eacf70a298ca2d98357f2e1cfc7bc07eb4af99 Mon Sep 17 00:00:00 2001 From: ycdesu Date: Fri, 5 Feb 2021 22:38:45 +0800 Subject: [PATCH 11/33] util: test Response struct --- pkg/util/http_response_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 pkg/util/http_response_test.go diff --git a/pkg/util/http_response_test.go b/pkg/util/http_response_test.go new file mode 100644 index 000000000..13fbe7897 --- /dev/null +++ b/pkg/util/http_response_test.go @@ -0,0 +1,28 @@ +package util + +import ( + "bytes" + "io/ioutil" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestResponse_DecodeJSON(t *testing.T) { + type temp struct { + Name string `json:"name"` + } + json := `{"name":"Test Name","a":"a"}` + reader := ioutil.NopCloser(bytes.NewReader([]byte(json))) + resp, err := NewResponse(&http.Response{ + StatusCode: 200, + Body: reader, + }) + assert.NoError(t, err) + assert.Equal(t, json, resp.String()) + + var result temp + assert.NoError(t, resp.DecodeJSON(&result)) + assert.Equal(t, "Test Name", result.Name) +} From 483f6270f40f38e2e3dd7cbccc4ad2fcd5a78afb Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 14:44:29 +0800 Subject: [PATCH 12/33] .travis.yml: add install section --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 595d79af6..4123aafac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,11 @@ go: - 1.15 services: - redis-server + +install: +- go get github.com/c9s/rockhopper/cmd/rockhopper + before_script: -- go get github.com/c9s/rockhopper - go mod download script: - bash scripts/test-sqlite3-migrations.sh From 256fce484c430b195bb74bbd5062b61a1915d2e5 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:15:03 +0800 Subject: [PATCH 13/33] add default 0 to not null field --- migrations/sqlite3/20201103173342_trades_add_order_id.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/sqlite3/20201103173342_trades_add_order_id.sql b/migrations/sqlite3/20201103173342_trades_add_order_id.sql index 7ff284200..8f0f94675 100644 --- a/migrations/sqlite3/20201103173342_trades_add_order_id.sql +++ b/migrations/sqlite3/20201103173342_trades_add_order_id.sql @@ -1,5 +1,5 @@ -- +up -ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER NOT NULL; +ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER NOT NULL DEFATUL 0; -- +down ALTER TABLE `trades` RENAME COLUMN `order_id` TO `order_id_deleted`; From 2691d06f5f3e8688129085d7ed68b14abed54e74 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:15:25 +0800 Subject: [PATCH 14/33] add rockhopper config for mysql --- rockhopper_mysql.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 rockhopper_mysql.yaml diff --git a/rockhopper_mysql.yaml b/rockhopper_mysql.yaml new file mode 100644 index 000000000..01e86e7bf --- /dev/null +++ b/rockhopper_mysql.yaml @@ -0,0 +1,5 @@ +--- +driver: mysql +dialect: mysql +dsn: "root@tcp(localhost:3306)/bbgo?parseTime=true" +migrationsDir: migrations From 6d756d2409644f62ae4ee17d5addc2baad383ce7 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:16:04 +0800 Subject: [PATCH 15/33] .travis.yml: add mysql to travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4123aafac..7d8e3dc53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,10 @@ language: go go: - 1.14 - 1.15 + services: - redis-server +- mysql install: - go get github.com/c9s/rockhopper/cmd/rockhopper From 94dc291580502da90064aa8c1f768fa918e78da6 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:16:56 +0800 Subject: [PATCH 16/33] .travis.yml: create bbgo mysql database --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7d8e3dc53..3d513d589 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,15 @@ services: - redis-server - mysql +before_install: +- mysql -e 'CREATE DATABASE bbgo;' + install: - go get github.com/c9s/rockhopper/cmd/rockhopper before_script: - go mod download + script: - bash scripts/test-sqlite3-migrations.sh - go test -v ./pkg/... From de51eb29e481988c0893f410e15a1166caf4c95a Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:48:05 +0800 Subject: [PATCH 17/33] refactor db stuff with database service --- pkg/bbgo/environment.go | 22 +++++------- pkg/cmd/backtest.go | 2 +- pkg/cmd/cancel.go | 7 ++-- pkg/cmd/run.go | 2 +- pkg/cmd/sync.go | 2 +- pkg/server/routes.go | 5 +-- pkg/server/setup.go | 2 +- pkg/service/database.go | 80 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 98 insertions(+), 24 deletions(-) create mode 100644 pkg/service/database.go diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index 11780b9ae..c03c8a711 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -7,7 +7,6 @@ import ( "time" "github.com/codingconcepts/env" - "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -48,6 +47,7 @@ type Environment struct { PersistenceServiceFacade *PersistenceServiceFacade + DatabaseService *service.DatabaseService OrderService *service.OrderService TradeService *service.TradeService TradeSync *service.SyncService @@ -56,8 +56,6 @@ type Environment struct { startTime time.Time tradeScanTime time.Time sessions map[string]*ExchangeSession - - MysqlURL string } func NewEnvironment() *Environment { @@ -78,23 +76,19 @@ func (environ *Environment) Sessions() map[string]*ExchangeSession { return environ.sessions } -func (environ *Environment) ConfigureDatabase(ctx context.Context, dsn string) error { - db, err := ConnectMySQL(dsn) +func (environ *Environment) ConfigureDatabase(ctx context.Context, driver string, dsn string) error { + environ.DatabaseService = service.NewDatabaseService(driver, dsn) + err := environ.DatabaseService.Connect() if err != nil { return err } - environ.MysqlURL = dsn - - if err := upgradeDB(ctx, "mysql", db.DB); err != nil { + if err := environ.DatabaseService.Upgrade(ctx) ; err != nil { return err } - environ.SetDB(db) - return nil -} - -func (environ *Environment) SetDB(db *sqlx.DB) *Environment { + // get the db connection pool object to create other services + db := environ.DatabaseService.DB environ.OrderService = &service.OrderService{DB: db} environ.TradeService = &service.TradeService{DB: db} environ.TradeSync = &service.SyncService{ @@ -102,7 +96,7 @@ func (environ *Environment) SetDB(db *sqlx.DB) *Environment { OrderService: environ.OrderService, } - return environ + return nil } // AddExchangeSession adds the existing exchange session or pre-created exchange session diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 0402c561b..638a4885d 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -118,7 +118,7 @@ var BacktestCmd = &cobra.Command{ environ := bbgo.NewEnvironment() if viper.IsSet("mysql-url") { dsn := viper.GetString("mysql-url") - if err := environ.ConfigureDatabase(ctx, dsn); err != nil { + if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { return err } } diff --git a/pkg/cmd/cancel.go b/pkg/cmd/cancel.go index 03cc64a41..f64971d8e 100644 --- a/pkg/cmd/cancel.go +++ b/pkg/cmd/cancel.go @@ -3,11 +3,11 @@ package cmd import ( "context" "fmt" + "os" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/types" @@ -64,12 +64,11 @@ var CancelCmd = &cobra.Command{ environ := bbgo.NewEnvironment() - if viper.IsSet("mysql-url") { - db, err := bbgo.ConnectMySQL(viper.GetString("mysql-url")) + if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { + err := environ.ConfigureDatabase(ctx, "mysql", dsn) if err != nil { return err } - environ.SetDB(db) } if err := environ.AddExchangesFromConfig(userConfig); err != nil { diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index b246d2d59..90eea1123 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -97,7 +97,7 @@ func runSetup(baseCtx context.Context, userConfig *bbgo.Config, enableApiServer func BootstrapEnvironment(ctx context.Context, environ *bbgo.Environment, userConfig *bbgo.Config) error { if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - if err := environ.ConfigureDatabase(ctx, dsn); err != nil { + if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { return err } } diff --git a/pkg/cmd/sync.go b/pkg/cmd/sync.go index e2d761fdf..8f0b0314f 100644 --- a/pkg/cmd/sync.go +++ b/pkg/cmd/sync.go @@ -55,7 +55,7 @@ var SyncCmd = &cobra.Command{ if viper.IsSet("mysql-url") { dsn := viper.GetString("mysql-url") - if err := environ.ConfigureDatabase(ctx, dsn); err != nil { + if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { return err } } diff --git a/pkg/server/routes.go b/pkg/server/routes.go index c70f62a46..5320b6ddc 100644 --- a/pkg/server/routes.go +++ b/pkg/server/routes.go @@ -421,8 +421,9 @@ func (s *Server) setupSaveConfig(c *gin.Context) { return } - if len(s.Environ.MysqlURL) > 0 { - envVars["MYSQL_URL"] = s.Environ.MysqlURL + if s.Environ.DatabaseService != nil { + envVars["DB_DRIVER"] = s.Environ.DatabaseService.Driver + envVars["DB_DSN"] = s.Environ.DatabaseService.DSN } dotenvFile := ".env.local" diff --git a/pkg/server/setup.go b/pkg/server/setup.go index 4232d812c..75d3a5788 100644 --- a/pkg/server/setup.go +++ b/pkg/server/setup.go @@ -58,7 +58,7 @@ func (s *Server) setupConfigureDB(c *gin.Context) { return } - if err := s.Environ.ConfigureDatabase(c, dsn); err != nil { + if err := s.Environ.ConfigureDatabase(c, "mysql", dsn); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } diff --git a/pkg/service/database.go b/pkg/service/database.go new file mode 100644 index 000000000..47d797b75 --- /dev/null +++ b/pkg/service/database.go @@ -0,0 +1,80 @@ +package service + +import ( + "context" + + "github.com/c9s/rockhopper" + "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" +) + +type DatabaseService struct { + Driver string + DSN string + DB *sqlx.DB +} + +func NewDatabaseService(driver, dsn string) *DatabaseService { + if driver == "mysql" { + var err error + dsn, err = ReformatMysqlDSN(dsn) + if err != nil { + // incorrect mysql dsn is logical exception + panic(err) + } + } + + return &DatabaseService{ + Driver: driver, + DSN: dsn, + } + +} + +func (s *DatabaseService) Connect() error { + var err error + s.DB, err = sqlx.Connect(s.Driver, s.DSN) + return err +} + +func (s *DatabaseService) Close() error { + return s.DB.Close() +} + +func (s *DatabaseService) Upgrade(ctx context.Context) error { + dialect, err := rockhopper.LoadDialect(s.Driver) + if err != nil { + return err + } + + loader := &rockhopper.GoMigrationLoader{} + migrations, err := loader.Load() + if err != nil { + return err + } + + // sqlx.DB is different from sql.DB + rh := rockhopper.New(s.Driver, dialect, s.DB.DB) + + currentVersion, err := rh.CurrentVersion() + if err != nil { + return err + } + + if err := rockhopper.Up(ctx, rh, migrations, currentVersion, 0); err != nil { + return err + } + + return nil +} + +func ReformatMysqlDSN(dsn string) (string, error) { + config, err := mysql.ParseDSN(dsn) + if err != nil { + return "", err + } + + config.ParseTime = true + dsn = config.FormatDSN() + return dsn, nil +} From 786757a686e78529a3e5ebe76e6918f1a4d9c896 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:49:05 +0800 Subject: [PATCH 18/33] migrations/sqlite3: fix sqlite3 alter syntax --- migrations/sqlite3/20201103173342_trades_add_order_id.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/sqlite3/20201103173342_trades_add_order_id.sql b/migrations/sqlite3/20201103173342_trades_add_order_id.sql index 8f0f94675..f337eb231 100644 --- a/migrations/sqlite3/20201103173342_trades_add_order_id.sql +++ b/migrations/sqlite3/20201103173342_trades_add_order_id.sql @@ -1,5 +1,5 @@ -- +up -ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER NOT NULL DEFATUL 0; +ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER; -- +down ALTER TABLE `trades` RENAME COLUMN `order_id` TO `order_id_deleted`; From 32f74d8f6f5733f6ecde749ca4d88c8248571155 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:50:02 +0800 Subject: [PATCH 19/33] move mysql migrations to migrations/mysql --- migrations/{ => mysql}/20200721225616_trades.sql | 0 migrations/{ => mysql}/20200819054742_trade_index.sql | 0 migrations/{ => mysql}/20201102222546_orders.sql | 0 migrations/{ => mysql}/20201103173342_trades_add_order_id.sql | 0 migrations/{ => mysql}/20201105092857_trades_index_fix.sql | 0 migrations/{ => mysql}/20201105093056_orders_add_index.sql | 0 migrations/{ => mysql}/20201106114742_klines.sql | 0 migrations/{ => mysql}/20201211175751_fix_symbol_length.sql | 0 migrations/{ => mysql}/20210118163847_fix_unique_index.sql | 0 migrations/{ => mysql}/20210119232826_add_margin_columns.sql | 0 .../{ => mysql}/20210129182704_trade_price_quantity_index.sql | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename migrations/{ => mysql}/20200721225616_trades.sql (100%) rename migrations/{ => mysql}/20200819054742_trade_index.sql (100%) rename migrations/{ => mysql}/20201102222546_orders.sql (100%) rename migrations/{ => mysql}/20201103173342_trades_add_order_id.sql (100%) rename migrations/{ => mysql}/20201105092857_trades_index_fix.sql (100%) rename migrations/{ => mysql}/20201105093056_orders_add_index.sql (100%) rename migrations/{ => mysql}/20201106114742_klines.sql (100%) rename migrations/{ => mysql}/20201211175751_fix_symbol_length.sql (100%) rename migrations/{ => mysql}/20210118163847_fix_unique_index.sql (100%) rename migrations/{ => mysql}/20210119232826_add_margin_columns.sql (100%) rename migrations/{ => mysql}/20210129182704_trade_price_quantity_index.sql (100%) diff --git a/migrations/20200721225616_trades.sql b/migrations/mysql/20200721225616_trades.sql similarity index 100% rename from migrations/20200721225616_trades.sql rename to migrations/mysql/20200721225616_trades.sql diff --git a/migrations/20200819054742_trade_index.sql b/migrations/mysql/20200819054742_trade_index.sql similarity index 100% rename from migrations/20200819054742_trade_index.sql rename to migrations/mysql/20200819054742_trade_index.sql diff --git a/migrations/20201102222546_orders.sql b/migrations/mysql/20201102222546_orders.sql similarity index 100% rename from migrations/20201102222546_orders.sql rename to migrations/mysql/20201102222546_orders.sql diff --git a/migrations/20201103173342_trades_add_order_id.sql b/migrations/mysql/20201103173342_trades_add_order_id.sql similarity index 100% rename from migrations/20201103173342_trades_add_order_id.sql rename to migrations/mysql/20201103173342_trades_add_order_id.sql diff --git a/migrations/20201105092857_trades_index_fix.sql b/migrations/mysql/20201105092857_trades_index_fix.sql similarity index 100% rename from migrations/20201105092857_trades_index_fix.sql rename to migrations/mysql/20201105092857_trades_index_fix.sql diff --git a/migrations/20201105093056_orders_add_index.sql b/migrations/mysql/20201105093056_orders_add_index.sql similarity index 100% rename from migrations/20201105093056_orders_add_index.sql rename to migrations/mysql/20201105093056_orders_add_index.sql diff --git a/migrations/20201106114742_klines.sql b/migrations/mysql/20201106114742_klines.sql similarity index 100% rename from migrations/20201106114742_klines.sql rename to migrations/mysql/20201106114742_klines.sql diff --git a/migrations/20201211175751_fix_symbol_length.sql b/migrations/mysql/20201211175751_fix_symbol_length.sql similarity index 100% rename from migrations/20201211175751_fix_symbol_length.sql rename to migrations/mysql/20201211175751_fix_symbol_length.sql diff --git a/migrations/20210118163847_fix_unique_index.sql b/migrations/mysql/20210118163847_fix_unique_index.sql similarity index 100% rename from migrations/20210118163847_fix_unique_index.sql rename to migrations/mysql/20210118163847_fix_unique_index.sql diff --git a/migrations/20210119232826_add_margin_columns.sql b/migrations/mysql/20210119232826_add_margin_columns.sql similarity index 100% rename from migrations/20210119232826_add_margin_columns.sql rename to migrations/mysql/20210119232826_add_margin_columns.sql diff --git a/migrations/20210129182704_trade_price_quantity_index.sql b/migrations/mysql/20210129182704_trade_price_quantity_index.sql similarity index 100% rename from migrations/20210129182704_trade_price_quantity_index.sql rename to migrations/mysql/20210129182704_trade_price_quantity_index.sql From c7440a3ea40bc8859e4686159251b13d5b5777ac Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:51:05 +0800 Subject: [PATCH 20/33] compile and update migration package --- pkg/migrations/mysql/20200721225616_trades.go | 33 ++++++++ .../mysql/20200819054742_trade_index.go | 53 ++++++++++++ pkg/migrations/mysql/20201102222546_orders.go | 33 ++++++++ .../20201103173342_trades_add_order_id.go | 33 ++++++++ .../mysql/20201105092857_trades_index_fix.go | 83 +++++++++++++++++++ .../mysql/20201105093056_orders_add_index.go | 43 ++++++++++ pkg/migrations/mysql/20201106114742_klines.go | 73 ++++++++++++++++ .../mysql/20201211175751_fix_symbol_length.go | 43 ++++++++++ .../mysql/20210118163847_fix_unique_index.go | 43 ++++++++++ .../20210119232826_add_margin_columns.go | 43 ++++++++++ ...210129182704_trade_price_quantity_index.go | 33 ++++++++ .../sqlite3/20200721225616_trades.go | 33 ++++++++ .../sqlite3/20200819054742_trade_index.go | 53 ++++++++++++ .../sqlite3/20201102222546_orders.go | 33 ++++++++ .../20201103173342_trades_add_order_id.go | 33 ++++++++ .../20201105092857_trades_index_fix.go | 83 +++++++++++++++++++ .../20201105093056_orders_add_index.go | 43 ++++++++++ .../sqlite3/20201106114742_klines.go | 73 ++++++++++++++++ .../20201211175751_fix_symbol_length.go | 33 ++++++++ .../20210118163847_fix_unique_index.go | 33 ++++++++ .../20210119232826_add_margin_columns.go | 63 ++++++++++++++ ...210129182704_trade_price_quantity_index.go | 33 ++++++++ 22 files changed, 1026 insertions(+) create mode 100644 pkg/migrations/mysql/20200721225616_trades.go create mode 100644 pkg/migrations/mysql/20200819054742_trade_index.go create mode 100644 pkg/migrations/mysql/20201102222546_orders.go create mode 100644 pkg/migrations/mysql/20201103173342_trades_add_order_id.go create mode 100644 pkg/migrations/mysql/20201105092857_trades_index_fix.go create mode 100644 pkg/migrations/mysql/20201105093056_orders_add_index.go create mode 100644 pkg/migrations/mysql/20201106114742_klines.go create mode 100644 pkg/migrations/mysql/20201211175751_fix_symbol_length.go create mode 100644 pkg/migrations/mysql/20210118163847_fix_unique_index.go create mode 100644 pkg/migrations/mysql/20210119232826_add_margin_columns.go create mode 100644 pkg/migrations/mysql/20210129182704_trade_price_quantity_index.go create mode 100644 pkg/migrations/sqlite3/20200721225616_trades.go create mode 100644 pkg/migrations/sqlite3/20200819054742_trade_index.go create mode 100644 pkg/migrations/sqlite3/20201102222546_orders.go create mode 100644 pkg/migrations/sqlite3/20201103173342_trades_add_order_id.go create mode 100644 pkg/migrations/sqlite3/20201105092857_trades_index_fix.go create mode 100644 pkg/migrations/sqlite3/20201105093056_orders_add_index.go create mode 100644 pkg/migrations/sqlite3/20201106114742_klines.go create mode 100644 pkg/migrations/sqlite3/20201211175751_fix_symbol_length.go create mode 100644 pkg/migrations/sqlite3/20210118163847_fix_unique_index.go create mode 100644 pkg/migrations/sqlite3/20210119232826_add_margin_columns.go create mode 100644 pkg/migrations/sqlite3/20210129182704_trade_price_quantity_index.go diff --git a/pkg/migrations/mysql/20200721225616_trades.go b/pkg/migrations/mysql/20200721225616_trades.go new file mode 100644 index 000000000..094b205ea --- /dev/null +++ b/pkg/migrations/mysql/20200721225616_trades.go @@ -0,0 +1,33 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTrades, downTrades) +} + +func upTrades(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `trades`\n(\n `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n `id` BIGINT UNSIGNED,\n `exchange` VARCHAR(24) NOT NULL DEFAULT '',\n `symbol` VARCHAR(8) NOT NULL,\n `price` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `quantity` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `quote_quantity` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `fee` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `fee_currency` VARCHAR(4) NOT NULL,\n `is_buyer` BOOLEAN NOT NULL DEFAULT FALSE,\n `is_maker` BOOLEAN NOT NULL DEFAULT FALSE,\n `side` VARCHAR(4) NOT NULL DEFAULT '',\n `traded_at` DATETIME(3) NOT NULL,\n PRIMARY KEY (`gid`),\n UNIQUE KEY `id` (`id`)\n);") + if err != nil { + return err + } + + return err +} + +func downTrades(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `trades`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20200819054742_trade_index.go b/pkg/migrations/mysql/20200819054742_trade_index.go new file mode 100644 index 000000000..e734dd88c --- /dev/null +++ b/pkg/migrations/mysql/20200819054742_trade_index.go @@ -0,0 +1,53 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradeIndex, downTradeIndex) +} + +func upTradeIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades(symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades(symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades(traded_at, symbol);") + if err != nil { + return err + } + + return err +} + +func downTradeIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol_fee_currency ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_traded_at_symbol ON trades;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201102222546_orders.go b/pkg/migrations/mysql/20201102222546_orders.go new file mode 100644 index 000000000..04aaa0653 --- /dev/null +++ b/pkg/migrations/mysql/20201102222546_orders.go @@ -0,0 +1,33 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upOrders, downOrders) +} + +func upOrders(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `orders`\n(\n `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n `exchange` VARCHAR(24) NOT NULL DEFAULT '',\n -- order_id is the order id returned from the exchange\n `order_id` BIGINT UNSIGNED NOT NULL,\n `client_order_id` VARCHAR(42) NOT NULL DEFAULT '',\n `order_type` VARCHAR(16) NOT NULL,\n `symbol` VARCHAR(8) NOT NULL,\n `status` VARCHAR(12) NOT NULL,\n `time_in_force` VARCHAR(4) NOT NULL,\n `price` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `stop_price` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `quantity` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `executed_quantity` DECIMAL(16, 8) UNSIGNED NOT NULL DEFAULT 0.0,\n `side` VARCHAR(4) NOT NULL DEFAULT '',\n `is_working` BOOL NOT NULL DEFAULT FALSE,\n `created_at` DATETIME(3) NOT NULL,\n `updated_at` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),\n PRIMARY KEY (`gid`)\n);") + if err != nil { + return err + } + + return err +} + +func downOrders(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE `orders`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201103173342_trades_add_order_id.go b/pkg/migrations/mysql/20201103173342_trades_add_order_id.go new file mode 100644 index 000000000..6b6dea539 --- /dev/null +++ b/pkg/migrations/mysql/20201103173342_trades_add_order_id.go @@ -0,0 +1,33 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradesAddOrderId, downTradesAddOrderId) +} + +func upTradesAddOrderId(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades`\n ADD COLUMN `order_id` BIGINT UNSIGNED NOT NULL;") + if err != nil { + return err + } + + return err +} + +func downTradesAddOrderId(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades`\n DROP COLUMN `order_id`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201105092857_trades_index_fix.go b/pkg/migrations/mysql/20201105092857_trades_index_fix.go new file mode 100644 index 000000000..d5a0917f1 --- /dev/null +++ b/pkg/migrations/mysql/20201105092857_trades_index_fix.go @@ -0,0 +1,83 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradesIndexFix, downTradesIndexFix) +} + +func upTradesIndexFix(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol_fee_currency ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_traded_at_symbol ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades (exchange, symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades (exchange, symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades (exchange, traded_at, symbol);") + if err != nil { + return err + } + + return err +} + +func downTradesIndexFix(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol_fee_currency ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_traded_at_symbol ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades (symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades (symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades (traded_at, symbol);") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201105093056_orders_add_index.go b/pkg/migrations/mysql/20201105093056_orders_add_index.go new file mode 100644 index 000000000..5e13521c5 --- /dev/null +++ b/pkg/migrations/mysql/20201105093056_orders_add_index.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upOrdersAddIndex, downOrdersAddIndex) +} + +func upOrdersAddIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX orders_symbol ON orders (exchange, symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE UNIQUE INDEX orders_order_id ON orders (order_id, exchange);") + if err != nil { + return err + } + + return err +} + +func downOrdersAddIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX orders_symbol ON orders;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX orders_order_id ON orders;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201106114742_klines.go b/pkg/migrations/mysql/20201106114742_klines.go new file mode 100644 index 000000000..0f5a8e756 --- /dev/null +++ b/pkg/migrations/mysql/20201106114742_klines.go @@ -0,0 +1,73 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upKlines, downKlines) +} + +func upKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `klines`\n(\n `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n `exchange` VARCHAR(10) NOT NULL,\n `start_time` DATETIME(3) NOT NULL,\n `end_time` DATETIME(3) NOT NULL,\n `interval` VARCHAR(3) NOT NULL,\n `symbol` VARCHAR(7) NOT NULL,\n `open` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `high` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `low` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `close` DECIMAL(16, 8) UNSIGNED NOT NULL DEFAULT 0.0,\n `volume` DECIMAL(16, 8) UNSIGNED NOT NULL DEFAULT 0.0,\n `closed` BOOL NOT NULL DEFAULT TRUE,\n `last_trade_id` INT UNSIGNED NOT NULL DEFAULT 0,\n `num_trades` INT UNSIGNED NOT NULL DEFAULT 0,\n PRIMARY KEY (`gid`)\n);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX `klines_end_time_symbol_interval` ON klines (`end_time`, `symbol`, `interval`);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `okex_klines` LIKE `klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `binance_klines` LIKE `klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `max_klines` LIKE `klines`;") + if err != nil { + return err + } + + return err +} + +func downKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX `klines_end_time_symbol_interval` ON `klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE `binance_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE `okex_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE `max_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE `klines`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201211175751_fix_symbol_length.go b/pkg/migrations/mysql/20201211175751_fix_symbol_length.go new file mode 100644 index 000000000..4bc7b15db --- /dev/null +++ b/pkg/migrations/mysql/20201211175751_fix_symbol_length.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upFixSymbolLength, downFixSymbolLength) +} + +func upFixSymbolLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE trades MODIFY COLUMN symbol VARCHAR(9);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE orders MODIFY COLUMN symbol VARCHAR(9);") + if err != nil { + return err + } + + return err +} + +func downFixSymbolLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE trades MODIFY COLUMN symbol VARCHAR(8);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE orders MODIFY COLUMN symbol VARCHAR(8);") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20210118163847_fix_unique_index.go b/pkg/migrations/mysql/20210118163847_fix_unique_index.go new file mode 100644 index 000000000..141fdf44e --- /dev/null +++ b/pkg/migrations/mysql/20210118163847_fix_unique_index.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upFixUniqueIndex, downFixUniqueIndex) +} + +func upFixUniqueIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` DROP INDEX `id`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` ADD UNIQUE INDEX `id` (`exchange`,`symbol`, `side`, `id`);") + if err != nil { + return err + } + + return err +} + +func downFixUniqueIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` DROP INDEX `id`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` ADD UNIQUE INDEX `id` (`id`);") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20210119232826_add_margin_columns.go b/pkg/migrations/mysql/20210119232826_add_margin_columns.go new file mode 100644 index 000000000..48a096f00 --- /dev/null +++ b/pkg/migrations/mysql/20210119232826_add_margin_columns.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upAddMarginColumns, downAddMarginColumns) +} + +func upAddMarginColumns(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades`\n ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE,\n ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE\n ;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders`\n ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE,\n ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE\n ;") + if err != nil { + return err + } + + return err +} + +func downAddMarginColumns(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades`\n DROP COLUMN `is_margin`,\n DROP COLUMN `is_isolated`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders`\n DROP COLUMN `is_margin`,\n DROP COLUMN `is_isolated`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20210129182704_trade_price_quantity_index.go b/pkg/migrations/mysql/20210129182704_trade_price_quantity_index.go new file mode 100644 index 000000000..22e5570aa --- /dev/null +++ b/pkg/migrations/mysql/20210129182704_trade_price_quantity_index.go @@ -0,0 +1,33 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradePriceQuantityIndex, downTradePriceQuantityIndex) +} + +func upTradePriceQuantityIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_price_quantity ON trades (order_id,price,quantity);") + if err != nil { + return err + } + + return err +} + +func downTradePriceQuantityIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_price_quantity ON trades") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20200721225616_trades.go b/pkg/migrations/sqlite3/20200721225616_trades.go new file mode 100644 index 000000000..d061db626 --- /dev/null +++ b/pkg/migrations/sqlite3/20200721225616_trades.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTrades, downTrades) +} + +func upTrades(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `trades`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `id` INTEGER,\n `exchange` TEXT NOT NULL DEFAULT '',\n `symbol` TEXT NOT NULL,\n `price` DECIMAL(16, 8) NOT NULL,\n `quantity` DECIMAL(16, 8) NOT NULL,\n `quote_quantity` DECIMAL(16, 8) NOT NULL,\n `fee` DECIMAL(16, 8) NOT NULL,\n `fee_currency` VARCHAR(4) NOT NULL,\n `is_buyer` BOOLEAN NOT NULL DEFAULT FALSE,\n `is_maker` BOOLEAN NOT NULL DEFAULT FALSE,\n `side` VARCHAR(4) NOT NULL DEFAULT '',\n `traded_at` DATETIME(3) NOT NULL\n);") + if err != nil { + return err + } + + return err +} + +func downTrades(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `trades`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20200819054742_trade_index.go b/pkg/migrations/sqlite3/20200819054742_trade_index.go new file mode 100644 index 000000000..c22577348 --- /dev/null +++ b/pkg/migrations/sqlite3/20200819054742_trade_index.go @@ -0,0 +1,53 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradeIndex, downTradeIndex) +} + +func upTradeIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades(symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades(symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades(traded_at, symbol);") + if err != nil { + return err + } + + return err +} + +func downTradeIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol_fee_currency;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_traded_at_symbol;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201102222546_orders.go b/pkg/migrations/sqlite3/20201102222546_orders.go new file mode 100644 index 000000000..b360c575d --- /dev/null +++ b/pkg/migrations/sqlite3/20201102222546_orders.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upOrders, downOrders) +} + +func upOrders(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `orders`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `exchange` VARCHAR NOT NULL DEFAULT '',\n -- order_id is the order id returned from the exchange\n `order_id` INTEGER NOT NULL,\n `client_order_id` VARCHAR NOT NULL DEFAULT '',\n `order_type` VARCHAR NOT NULL,\n `symbol` VARCHAR NOT NULL,\n `status` VARCHAR NOT NULL,\n `time_in_force` VARCHAR NOT NULL,\n `price` DECIMAL(16, 8) NOT NULL,\n `stop_price` DECIMAL(16, 8) NOT NULL,\n `quantity` DECIMAL(16, 8) NOT NULL,\n `executed_quantity` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `side` VARCHAR NOT NULL DEFAULT '',\n `is_working` BOOLEAN NOT NULL DEFAULT FALSE,\n `created_at` DATETIME(3) NOT NULL,\n `updated_at` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP\n);") + if err != nil { + return err + } + + return err +} + +func downOrders(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `orders`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201103173342_trades_add_order_id.go b/pkg/migrations/sqlite3/20201103173342_trades_add_order_id.go new file mode 100644 index 000000000..7130b78b6 --- /dev/null +++ b/pkg/migrations/sqlite3/20201103173342_trades_add_order_id.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradesAddOrderId, downTradesAddOrderId) +} + +func upTradesAddOrderId(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER;") + if err != nil { + return err + } + + return err +} + +func downTradesAddOrderId(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` RENAME COLUMN `order_id` TO `order_id_deleted`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201105092857_trades_index_fix.go b/pkg/migrations/sqlite3/20201105092857_trades_index_fix.go new file mode 100644 index 000000000..05112484d --- /dev/null +++ b/pkg/migrations/sqlite3/20201105092857_trades_index_fix.go @@ -0,0 +1,83 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradesIndexFix, downTradesIndexFix) +} + +func upTradesIndexFix(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_symbol_fee_currency;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_traded_at_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades (exchange, symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades (exchange, symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades (exchange, traded_at, symbol);") + if err != nil { + return err + } + + return err +} + +func downTradesIndexFix(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_symbol_fee_currency;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_traded_at_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades (symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades (symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades (traded_at, symbol);") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201105093056_orders_add_index.go b/pkg/migrations/sqlite3/20201105093056_orders_add_index.go new file mode 100644 index 000000000..d880d2d0a --- /dev/null +++ b/pkg/migrations/sqlite3/20201105093056_orders_add_index.go @@ -0,0 +1,43 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upOrdersAddIndex, downOrdersAddIndex) +} + +func upOrdersAddIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX orders_symbol ON orders (exchange, symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE UNIQUE INDEX orders_order_id ON orders (order_id, exchange);") + if err != nil { + return err + } + + return err +} + +func downOrdersAddIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS orders_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS orders_order_id;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201106114742_klines.go b/pkg/migrations/sqlite3/20201106114742_klines.go new file mode 100644 index 000000000..792cc7183 --- /dev/null +++ b/pkg/migrations/sqlite3/20201106114742_klines.go @@ -0,0 +1,73 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upKlines, downKlines) +} + +func upKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `klines`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `exchange` VARCHAR(10) NOT NULL,\n `start_time` DATETIME(3) NOT NULL,\n `end_time` DATETIME(3) NOT NULL,\n `interval` VARCHAR(3) NOT NULL,\n `symbol` VARCHAR(7) NOT NULL,\n `open` DECIMAL(16, 8) NOT NULL,\n `high` DECIMAL(16, 8) NOT NULL,\n `low` DECIMAL(16, 8) NOT NULL,\n `close` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `volume` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `closed` BOOLEAN NOT NULL DEFAULT TRUE,\n `last_trade_id` INT NOT NULL DEFAULT 0,\n `num_trades` INT NOT NULL DEFAULT 0\n);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX `klines_end_time_symbol_interval` ON klines (`end_time`, `symbol`, `interval`);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `okex_klines` AS SELECT * FROM `klines` WHERE 0") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `binance_klines` AS SELECT * FROM `klines` WHERE 0") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `max_klines` AS SELECT * FROM `klines` WHERE 0") + if err != nil { + return err + } + + return err +} + +func downKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS `klines_end_time_symbol_interval`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `binance_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `okex_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `max_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `klines`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201211175751_fix_symbol_length.go b/pkg/migrations/sqlite3/20201211175751_fix_symbol_length.go new file mode 100644 index 000000000..562967063 --- /dev/null +++ b/pkg/migrations/sqlite3/20201211175751_fix_symbol_length.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upFixSymbolLength, downFixSymbolLength) +} + +func upFixSymbolLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "SELECT 1;") + if err != nil { + return err + } + + return err +} + +func downFixSymbolLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "SELECT 1;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20210118163847_fix_unique_index.go b/pkg/migrations/sqlite3/20210118163847_fix_unique_index.go new file mode 100644 index 000000000..87b1a8a91 --- /dev/null +++ b/pkg/migrations/sqlite3/20210118163847_fix_unique_index.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upFixUniqueIndex, downFixUniqueIndex) +} + +func upFixUniqueIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE UNIQUE INDEX `trade_unique_id` ON `trades` (`exchange`,`symbol`, `side`, `id`);") + if err != nil { + return err + } + + return err +} + +func downFixUniqueIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS `trade_unique_id`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20210119232826_add_margin_columns.go b/pkg/migrations/sqlite3/20210119232826_add_margin_columns.go new file mode 100644 index 000000000..f403e5ad6 --- /dev/null +++ b/pkg/migrations/sqlite3/20210119232826_add_margin_columns.go @@ -0,0 +1,63 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upAddMarginColumns, downAddMarginColumns) +} + +func upAddMarginColumns(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders` ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders` ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE;") + if err != nil { + return err + } + + return err +} + +func downAddMarginColumns(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` RENAME COLUMN `is_margin` TO `is_margin_deleted`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` RENAME COLUMN `is_isolated` TO `is_isolated_deleted`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders` RENAME COLUMN `is_margin` TO `is_margin_deleted`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders` RENAME COLUMN `is_isolated` TO `is_isolated_deleted`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20210129182704_trade_price_quantity_index.go b/pkg/migrations/sqlite3/20210129182704_trade_price_quantity_index.go new file mode 100644 index 000000000..73c608a61 --- /dev/null +++ b/pkg/migrations/sqlite3/20210129182704_trade_price_quantity_index.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradePriceQuantityIndex, downTradePriceQuantityIndex) +} + +func upTradePriceQuantityIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_price_quantity ON trades (order_id,price,quantity);") + if err != nil { + return err + } + + return err +} + +func downTradePriceQuantityIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_price_quantity;") + if err != nil { + return err + } + + return err +} From 63a8c00a4c3cb1fdfc451fc528e0af860873d95c Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:51:11 +0800 Subject: [PATCH 21/33] fix makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9a6ff5d5f..8dbbd46e4 100644 --- a/Makefile +++ b/Makefile @@ -28,9 +28,9 @@ dist: bin-dir bbgo-linux bbgo-darwin tar -C $(BUILD_DIR) -cvzf $(DIST_DIR)/bbgo-$$(git describe --tags).tar.gz . migrations: - rockhopper compile --config rockhopper.yaml --output pkg/migrations - git add -v pkg/migrations - git commit -m "Update migration package" pkg/migrations + rockhopper compile --config rockhopper_mysql.yaml --output pkg/migrations/mysql + rockhopper compile --config rockhopper_sqlite.yaml --output pkg/migrations/sqlite3 + git add -v pkg/migrations && git commit -m "compile and update migration package" pkg/migrations || true docker: GOPATH=$(PWD)/_mod go mod download From ecf94cdeea9f015e6afc485505129e6f86e7ddfe Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:51:35 +0800 Subject: [PATCH 22/33] .travis.yml: test make migrations --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3d513d589..427b4b326 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ install: before_script: - go mod download +- make migrations script: - bash scripts/test-sqlite3-migrations.sh From 276b6c1e48da226a1c863d78c0f1d122791a1354 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:52:53 +0800 Subject: [PATCH 23/33] drop the legacy upgradeDB --- pkg/bbgo/db.go | 29 ----------------------------- rockhopper_mysql.yaml | 2 +- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/pkg/bbgo/db.go b/pkg/bbgo/db.go index 0ff4f0d13..50431f02c 100644 --- a/pkg/bbgo/db.go +++ b/pkg/bbgo/db.go @@ -1,13 +1,9 @@ package bbgo import ( - "context" - "database/sql" - // register the go migrations _ "github.com/c9s/bbgo/pkg/migrations" - "github.com/c9s/rockhopper" "github.com/go-sql-driver/mysql" "github.com/jmoiron/sqlx" ) @@ -23,28 +19,3 @@ func ConnectMySQL(dsn string) (*sqlx.DB, error) { return sqlx.Connect("mysql", dsn) } -func upgradeDB(ctx context.Context, driver string, db *sql.DB) error { - dialect, err := rockhopper.LoadDialect(driver) - if err != nil { - return err - } - - loader := &rockhopper.GoMigrationLoader{} - migrations, err := loader.Load() - if err != nil { - return err - } - - rh := rockhopper.New(driver, dialect, db) - - currentVersion, err := rh.CurrentVersion() - if err != nil { - return err - } - - if err := rockhopper.Up(ctx, rh, migrations, currentVersion, 0); err != nil { - return err - } - - return nil -} diff --git a/rockhopper_mysql.yaml b/rockhopper_mysql.yaml index 01e86e7bf..6dde3ff23 100644 --- a/rockhopper_mysql.yaml +++ b/rockhopper_mysql.yaml @@ -2,4 +2,4 @@ driver: mysql dialect: mysql dsn: "root@tcp(localhost:3306)/bbgo?parseTime=true" -migrationsDir: migrations +migrationsDir: migrations/mysql From 99b56003ebebf68716b014572f04c6a082ae2e25 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:56:26 +0800 Subject: [PATCH 24/33] clean up legacy db connection handling with the new database service --- pkg/bbgo/db.go | 15 --------------- pkg/cmd/backtest.go | 14 +++++--------- pkg/cmd/pnl.go | 18 ++++++++++-------- pkg/server/setup.go | 4 ++-- 4 files changed, 17 insertions(+), 34 deletions(-) diff --git a/pkg/bbgo/db.go b/pkg/bbgo/db.go index 50431f02c..dcdb0eaec 100644 --- a/pkg/bbgo/db.go +++ b/pkg/bbgo/db.go @@ -3,19 +3,4 @@ package bbgo import ( // register the go migrations _ "github.com/c9s/bbgo/pkg/migrations" - - "github.com/go-sql-driver/mysql" - "github.com/jmoiron/sqlx" ) - -func ConnectMySQL(dsn string) (*sqlx.DB, error) { - config, err := mysql.ParseDSN(dsn) - if err != nil { - return nil, err - } - - config.ParseTime = true - dsn = config.FormatDSN() - return sqlx.Connect("mysql", dsn) -} - diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 638a4885d..8816302d3 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -3,12 +3,12 @@ package cmd import ( "context" "fmt" + "os" "time" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/c9s/bbgo/pkg/accounting/pnl" "github.com/c9s/bbgo/pkg/backtest" @@ -96,10 +96,6 @@ var BacktestCmd = &cobra.Command{ return err } - db, err := bbgo.ConnectMySQL(viper.GetString("mysql-url")) - if err != nil { - return err - } if userConfig.Backtest == nil { return errors.New("backtest config is not defined") @@ -116,14 +112,14 @@ var BacktestCmd = &cobra.Command{ } environ := bbgo.NewEnvironment() - if viper.IsSet("mysql-url") { - dsn := viper.GetString("mysql-url") - if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { + if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { + err := environ.ConfigureDatabase(ctx, "mysql", dsn) + if err != nil { return err } } - backtestService := &service.BacktestService{DB: db} + backtestService := &service.BacktestService{DB: environ.DatabaseService.DB} if wantSync { log.Info("starting synchronization...") diff --git a/pkg/cmd/pnl.go b/pkg/cmd/pnl.go index 9e27c6f5b..82048a10e 100644 --- a/pkg/cmd/pnl.go +++ b/pkg/cmd/pnl.go @@ -2,13 +2,13 @@ package cmd import ( "context" + "os" "strings" "time" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/c9s/bbgo/pkg/accounting" "github.com/c9s/bbgo/pkg/accounting/pnl" @@ -51,20 +51,22 @@ var PnLCmd = &cobra.Command{ return err } - db, err := bbgo.ConnectMySQL(viper.GetString("mysql-url")) - if err != nil { - return err - } - tradeService := &service.TradeService{DB: db} + environ := bbgo.NewEnvironment() + if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { + err := environ.ConfigureDatabase(ctx, "mysql", dsn) + if err != nil { + return err + } + } var trades []types.Trade tradingFeeCurrency := exchange.PlatformFeeCurrency() if strings.HasPrefix(symbol, tradingFeeCurrency) { log.Infof("loading all trading fee currency related trades: %s", symbol) - trades, err = tradeService.QueryForTradingFeeCurrency(exchange.Name(), symbol, tradingFeeCurrency) + trades, err = environ.TradeService.QueryForTradingFeeCurrency(exchange.Name(), symbol, tradingFeeCurrency) } else { - trades, err = tradeService.Query(service.QueryTradesOptions{ + trades, err = environ.TradeService.Query(service.QueryTradesOptions{ Exchange: exchange.Name(), Symbol: symbol, }) diff --git a/pkg/server/setup.go b/pkg/server/setup.go index 75d3a5788..6619429ad 100644 --- a/pkg/server/setup.go +++ b/pkg/server/setup.go @@ -2,6 +2,7 @@ package server import ( "context" + "database/sql" "net/http" "os" "syscall" @@ -29,7 +30,7 @@ func (s *Server) setupTestDB(c *gin.Context) { return } - db, err := bbgo.ConnectMySQL(dsn) + db, err := sql.Open("mysql", dsn) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return @@ -142,4 +143,3 @@ func (s *Server) setupRestart(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"success": true}) } - From 51d399a49d6e2e82f8768c3913124121db8f751d Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 11:17:31 +0800 Subject: [PATCH 25/33] upgrade rockhopper --- go.mod | 4 ++-- go.sum | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 4a4033a8a..8fb3096d3 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.13 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/adshao/go-binance/v2 v2.2.1-0.20210119141603-20ceb26d876b - github.com/c9s/rockhopper v1.2.1-0.20210115022144-cc77e66fc34f + github.com/c9s/rockhopper v1.2.1-0.20210206025705-bbb1e34bd7a9 github.com/codingconcepts/env v0.0.0-20200821220118-a8fbf8d84482 github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect github.com/gin-contrib/cors v1.3.1 @@ -53,7 +53,7 @@ require ( github.com/x-cray/logrus-prefixed-formatter v0.5.2 github.com/zserge/lorca v0.1.9 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect - golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect golang.org/x/text v0.3.5 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 gonum.org/v1/gonum v0.8.1 diff --git a/go.sum b/go.sum index 982d91dba..7db55f485 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8 github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/c9s/rockhopper v1.2.1-0.20210115022144-cc77e66fc34f h1:n1Ly7178MJj+GQB38q4dV66QktUvzEi2rA7xCtTy6Ck= github.com/c9s/rockhopper v1.2.1-0.20210115022144-cc77e66fc34f/go.mod h1:KJnQjZSrWA83jjwGF/+O7Y96VCVirYTYEvXJJOc6kMU= +github.com/c9s/rockhopper v1.2.1-0.20210206025705-bbb1e34bd7a9 h1:umJ5T1aKfA6zmHTJffe5axqM9mtr/tscllnX2wnZzBA= +github.com/c9s/rockhopper v1.2.1-0.20210206025705-bbb1e34bd7a9/go.mod h1:KJnQjZSrWA83jjwGF/+O7Y96VCVirYTYEvXJJOc6kMU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -240,6 +242,7 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -466,6 +469,8 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From dd9dbee90303f5b873eb6143b761e06bf4d6ae3e Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 11:33:49 +0800 Subject: [PATCH 26/33] refactor database configuration with env vars --- pkg/cmd/backtest.go | 29 ++++++++++++++++++++++++----- pkg/cmd/cancel.go | 9 ++------- pkg/cmd/pnl.go | 8 ++------ pkg/cmd/run.go | 6 ++---- pkg/cmd/sync.go | 9 ++------- pkg/server/setup.go | 3 ++- pkg/service/database.go | 3 ++- 7 files changed, 36 insertions(+), 31 deletions(-) diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 8816302d3..425a4be36 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -112,11 +112,9 @@ var BacktestCmd = &cobra.Command{ } environ := bbgo.NewEnvironment() - if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - err := environ.ConfigureDatabase(ctx, "mysql", dsn) - if err != nil { - return err - } + + if err := configureDB(ctx, environ) ; err != nil { + return err } backtestService := &service.BacktestService{DB: environ.DatabaseService.DB} @@ -289,3 +287,24 @@ func InBaseAsset(balances types.BalanceMap, market types.Market, price float64) base := balances[market.BaseCurrency] return (base.Locked.Float64() + base.Available.Float64()) + ((quote.Locked.Float64() + quote.Available.Float64()) / price) } + +// configureDB configures the database service based on the environment variable +func configureDB(ctx context.Context, environ *bbgo.Environment) error { + if driver, ok := os.LookupEnv("DB_DRIVER") ; ok { + + if dsn, ok := os.LookupEnv("DB_DSN") ; ok { + return environ.ConfigureDatabase(ctx, driver, dsn) + } + + } else if dsn, ok := os.LookupEnv("SQLITE3_DSN"); ok { + + return environ.ConfigureDatabase(ctx, "sqlite3", dsn) + + } else if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { + + return environ.ConfigureDatabase(ctx, "mysql", dsn) + + } + + return nil +} diff --git a/pkg/cmd/cancel.go b/pkg/cmd/cancel.go index f64971d8e..af4526587 100644 --- a/pkg/cmd/cancel.go +++ b/pkg/cmd/cancel.go @@ -3,7 +3,6 @@ package cmd import ( "context" "fmt" - "os" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -63,12 +62,8 @@ var CancelCmd = &cobra.Command{ } environ := bbgo.NewEnvironment() - - if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - err := environ.ConfigureDatabase(ctx, "mysql", dsn) - if err != nil { - return err - } + if err := configureDB(ctx, environ) ; err != nil { + return err } if err := environ.AddExchangesFromConfig(userConfig); err != nil { diff --git a/pkg/cmd/pnl.go b/pkg/cmd/pnl.go index 82048a10e..215a54ba6 100644 --- a/pkg/cmd/pnl.go +++ b/pkg/cmd/pnl.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "os" "strings" "time" @@ -53,11 +52,8 @@ var PnLCmd = &cobra.Command{ environ := bbgo.NewEnvironment() - if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - err := environ.ConfigureDatabase(ctx, "mysql", dsn) - if err != nil { - return err - } + if err := configureDB(ctx, environ) ; err != nil { + return err } var trades []types.Trade diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 90eea1123..7c042ff4a 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -96,10 +96,8 @@ func runSetup(baseCtx context.Context, userConfig *bbgo.Config, enableApiServer } func BootstrapEnvironment(ctx context.Context, environ *bbgo.Environment, userConfig *bbgo.Config) error { - if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { - return err - } + if err := configureDB(ctx, environ) ; err != nil { + return err } if err := environ.AddExchangesFromConfig(userConfig); err != nil { diff --git a/pkg/cmd/sync.go b/pkg/cmd/sync.go index 8f0b0314f..05bc867fd 100644 --- a/pkg/cmd/sync.go +++ b/pkg/cmd/sync.go @@ -9,7 +9,6 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/c9s/bbgo/pkg/bbgo" ) @@ -52,12 +51,8 @@ var SyncCmd = &cobra.Command{ } environ := bbgo.NewEnvironment() - - if viper.IsSet("mysql-url") { - dsn := viper.GetString("mysql-url") - if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { - return err - } + if err := configureDB(ctx, environ) ; err != nil { + return err } if err := environ.AddExchangesFromConfig(userConfig); err != nil { diff --git a/pkg/server/setup.go b/pkg/server/setup.go index 6619429ad..d81a7f9d4 100644 --- a/pkg/server/setup.go +++ b/pkg/server/setup.go @@ -59,7 +59,8 @@ func (s *Server) setupConfigureDB(c *gin.Context) { return } - if err := s.Environ.ConfigureDatabase(c, "mysql", dsn); err != nil { + driver := "mysql" + if err := s.Environ.ConfigureDatabase(c, driver, dsn); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } diff --git a/pkg/service/database.go b/pkg/service/database.go index 47d797b75..05f5b5bcb 100644 --- a/pkg/service/database.go +++ b/pkg/service/database.go @@ -48,7 +48,7 @@ func (s *DatabaseService) Upgrade(ctx context.Context) error { } loader := &rockhopper.GoMigrationLoader{} - migrations, err := loader.Load() + migrations, err := loader.LoadByPackageSuffix(s.Driver) if err != nil { return err } @@ -74,6 +74,7 @@ func ReformatMysqlDSN(dsn string) (string, error) { return "", err } + // we need timestamp and datetime fields to be parsed into time.Time struct config.ParseTime = true dsn = config.FormatDSN() return dsn, nil From 0b657d59f99243cb06e41331dd8a16cbd8a4654d Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 11:34:53 +0800 Subject: [PATCH 27/33] make inBaseAsset as private method --- pkg/cmd/backtest.go | 31 ++----------------------------- pkg/cmd/utils.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 29 deletions(-) create mode 100644 pkg/cmd/utils.go diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 425a4be36..f5765acf9 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -3,7 +3,6 @@ package cmd import ( "context" "fmt" - "os" "time" "github.com/pkg/errors" @@ -267,8 +266,8 @@ var BacktestCmd = &cobra.Command{ finalBalances.Print() if wantBaseAssetBaseline { - initBaseAsset := InBaseAsset(initBalances, market, startPrice) - finalBaseAsset := InBaseAsset(finalBalances, market, lastPrice) + initBaseAsset := inBaseAsset(initBalances, market, startPrice) + finalBaseAsset := inBaseAsset(finalBalances, market, lastPrice) log.Infof("INITIAL ASSET ~= %s %s (1 %s = %f)", market.FormatQuantity(initBaseAsset), market.BaseCurrency, market.BaseCurrency, startPrice) log.Infof("FINAL ASSET ~= %s %s (1 %s = %f)", market.FormatQuantity(finalBaseAsset), market.BaseCurrency, market.BaseCurrency, lastPrice) @@ -282,29 +281,3 @@ var BacktestCmd = &cobra.Command{ }, } -func InBaseAsset(balances types.BalanceMap, market types.Market, price float64) float64 { - quote := balances[market.QuoteCurrency] - base := balances[market.BaseCurrency] - return (base.Locked.Float64() + base.Available.Float64()) + ((quote.Locked.Float64() + quote.Available.Float64()) / price) -} - -// configureDB configures the database service based on the environment variable -func configureDB(ctx context.Context, environ *bbgo.Environment) error { - if driver, ok := os.LookupEnv("DB_DRIVER") ; ok { - - if dsn, ok := os.LookupEnv("DB_DSN") ; ok { - return environ.ConfigureDatabase(ctx, driver, dsn) - } - - } else if dsn, ok := os.LookupEnv("SQLITE3_DSN"); ok { - - return environ.ConfigureDatabase(ctx, "sqlite3", dsn) - - } else if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - - return environ.ConfigureDatabase(ctx, "mysql", dsn) - - } - - return nil -} diff --git a/pkg/cmd/utils.go b/pkg/cmd/utils.go new file mode 100644 index 000000000..9298cc659 --- /dev/null +++ b/pkg/cmd/utils.go @@ -0,0 +1,36 @@ +package cmd + +import ( + "context" + "os" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/types" +) + +func inBaseAsset(balances types.BalanceMap, market types.Market, price float64) float64 { + quote := balances[market.QuoteCurrency] + base := balances[market.BaseCurrency] + return (base.Locked.Float64() + base.Available.Float64()) + ((quote.Locked.Float64() + quote.Available.Float64()) / price) +} + +// configureDB configures the database service based on the environment variable +func configureDB(ctx context.Context, environ *bbgo.Environment) error { + if driver, ok := os.LookupEnv("DB_DRIVER"); ok { + + if dsn, ok := os.LookupEnv("DB_DSN"); ok { + return environ.ConfigureDatabase(ctx, driver, dsn) + } + + } else if dsn, ok := os.LookupEnv("SQLITE3_DSN"); ok { + + return environ.ConfigureDatabase(ctx, "sqlite3", dsn) + + } else if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { + + return environ.ConfigureDatabase(ctx, "mysql", dsn) + + } + + return nil +} From 32117af4b047779ddf6643ddc3648875bbbb97bd Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 11:44:49 +0800 Subject: [PATCH 28/33] service: remove the ignore keyword to make the sql compatible with sqlite3 --- pkg/service/trade.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/service/trade.go b/pkg/service/trade.go index 6739fbf07..e607cc668 100644 --- a/pkg/service/trade.go +++ b/pkg/service/trade.go @@ -244,7 +244,7 @@ func (s *TradeService) scanRows(rows *sqlx.Rows) (trades []types.Trade, err erro func (s *TradeService) Insert(trade types.Trade) error { _, err := s.DB.NamedExec(` - INSERT IGNORE INTO trades (id, exchange, order_id, symbol, price, quantity, quote_quantity, side, is_buyer, is_maker, fee, fee_currency, traded_at, is_margin, is_isolated) + INSERT INTO trades (id, exchange, order_id, symbol, price, quantity, quote_quantity, side, is_buyer, is_maker, fee, fee_currency, traded_at, is_margin, is_isolated) VALUES (:id, :exchange, :order_id, :symbol, :price, :quantity, :quote_quantity, :side, :is_buyer, :is_maker, :fee, :fee_currency, :traded_at, :is_margin, :is_isolated)`, trade) return err From 3abdb3dd7bfa6dc90803e584ad1fee0e14c21d4c Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 12:32:21 +0800 Subject: [PATCH 29/33] convert time struct for sqlite driver --- pkg/accounting/pnl/avg_cost.go | 3 +- pkg/backtest/matching.go | 3 +- pkg/datatype/time.go | 59 +++++++++++++++++++++++++++++++++ pkg/exchange/binance/convert.go | 3 +- pkg/exchange/binance/parse.go | 3 +- pkg/exchange/max/convert.go | 3 +- pkg/exchange/max/stream.go | 3 +- pkg/service/sync.go | 2 +- pkg/types/batch.go | 2 +- pkg/types/trade.go | 14 ++++---- 10 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 pkg/datatype/time.go diff --git a/pkg/accounting/pnl/avg_cost.go b/pkg/accounting/pnl/avg_cost.go index f05f49533..5be9d9de9 100644 --- a/pkg/accounting/pnl/avg_cost.go +++ b/pkg/accounting/pnl/avg_cost.go @@ -2,6 +2,7 @@ package pnl import ( "strings" + "time" "github.com/c9s/bbgo/pkg/types" ) @@ -96,7 +97,7 @@ func (c *AverageCostCalculator) Calculate(symbol string, trades []types.Trade, c Symbol: symbol, CurrentPrice: currentPrice, NumTrades: len(trades), - StartTime: trades[0].Time, + StartTime: time.Time(trades[0].Time), BuyVolume: bidVolume, SellVolume: askVolume, diff --git a/pkg/backtest/matching.go b/pkg/backtest/matching.go index 24b687a56..e54bc06a2 100644 --- a/pkg/backtest/matching.go +++ b/pkg/backtest/matching.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/c9s/bbgo/pkg/datatype" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -235,7 +236,7 @@ func (m *SimplePriceMatching) newTradeFromOrder(order types.Order, isMaker bool) Side: order.Side, IsBuyer: order.Side == types.SideTypeBuy, IsMaker: isMaker, - Time: m.CurrentTime, + Time: datatype.Time(m.CurrentTime), Fee: fee, FeeCurrency: feeCurrency, } diff --git a/pkg/datatype/time.go b/pkg/datatype/time.go new file mode 100644 index 000000000..e871c2273 --- /dev/null +++ b/pkg/datatype/time.go @@ -0,0 +1,59 @@ +package datatype + +import ( + "database/sql/driver" + "fmt" + "time" +) + +type Time time.Time + +var layout = "2006-01-02 15:04:05.999Z07:00" + +func (t Time) String() string { + return time.Time(t).String() +} + +// driver.Valuer interface +// see http://jmoiron.net/blog/built-in-interfaces/ +func (t Time) Value() (driver.Value, error) { + return time.Time(t), nil +} + +func (t *Time) Scan(src interface{}) error { + switch d := src.(type) { + + case *time.Time: + *t = Time(*d) + return nil + + case time.Time: + *t = Time(d) + return nil + + case string: + // 2020-12-16 05:17:12.994+08:00 + tt, err := time.Parse(layout, d) + if err != nil { + return err + } + + *t = Time(tt) + return nil + + case []byte: + // 2019-10-20 23:01:43.77+08:00 + tt, err := time.Parse(layout, string(d)) + if err != nil { + return err + } + + *t = Time(tt) + return nil + + default: + + } + + return fmt.Errorf("datatype.Time scan error, type: %T is not supported, value; %+v", src, src) +} diff --git a/pkg/exchange/binance/convert.go b/pkg/exchange/binance/convert.go index 41330cd61..a1185f4b0 100644 --- a/pkg/exchange/binance/convert.go +++ b/pkg/exchange/binance/convert.go @@ -8,6 +8,7 @@ import ( "github.com/adshao/go-binance/v2" "github.com/pkg/errors" + "github.com/c9s/bbgo/pkg/datatype" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" ) @@ -115,7 +116,7 @@ func ToGlobalTrade(t binance.TradeV3, isMargin bool) (*types.Trade, error) { Fee: fee, FeeCurrency: t.CommissionAsset, QuoteQuantity: quoteQuantity, - Time: millisecondTime(t.Time), + Time: datatype.Time(millisecondTime(t.Time)), IsMargin: isMargin, IsIsolated: t.IsIsolated, }, nil diff --git a/pkg/exchange/binance/parse.go b/pkg/exchange/binance/parse.go index 65dfac764..d983ca69c 100644 --- a/pkg/exchange/binance/parse.go +++ b/pkg/exchange/binance/parse.go @@ -9,6 +9,7 @@ import ( "github.com/adshao/go-binance/v2" "github.com/valyala/fastjson" + "github.com/c9s/bbgo/pkg/datatype" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" @@ -132,7 +133,7 @@ func (e *ExecutionReportEvent) Trade() (*types.Trade, error) { QuoteQuantity: util.MustParseFloat(e.LastQuoteAssetTransactedQuantity), IsBuyer: e.Side == "BUY", IsMaker: e.IsMaker, - Time: tt, + Time: datatype.Time(tt), Fee: util.MustParseFloat(e.CommissionAmount), FeeCurrency: e.CommissionAsset, }, nil diff --git a/pkg/exchange/max/convert.go b/pkg/exchange/max/convert.go index 6b38daa8a..0586f9e5b 100644 --- a/pkg/exchange/max/convert.go +++ b/pkg/exchange/max/convert.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/c9s/bbgo/pkg/datatype" "github.com/c9s/bbgo/pkg/exchange/max/maxapi" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -211,7 +212,7 @@ func toGlobalTrade(t max.Trade) (*types.Trade, error) { Fee: fee, FeeCurrency: toGlobalCurrency(t.FeeCurrency), QuoteQuantity: quoteQuantity, - Time: mts, + Time: datatype.Time(mts), }, nil } diff --git a/pkg/exchange/max/stream.go b/pkg/exchange/max/stream.go index 79c644212..67f6cc521 100644 --- a/pkg/exchange/max/stream.go +++ b/pkg/exchange/max/stream.go @@ -7,6 +7,7 @@ import ( "github.com/gorilla/websocket" + "github.com/c9s/bbgo/pkg/datatype" max "github.com/c9s/bbgo/pkg/exchange/max/maxapi" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -198,7 +199,7 @@ func convertWebSocketTrade(t max.TradeUpdate) (*types.Trade, error) { Fee: fee, FeeCurrency: toGlobalCurrency(t.FeeCurrency), QuoteQuantity: quoteQuantity, - Time: mts, + Time: datatype.Time(mts), }, nil } diff --git a/pkg/service/sync.go b/pkg/service/sync.go index c28890d97..a1d58e629 100644 --- a/pkg/service/sync.go +++ b/pkg/service/sync.go @@ -84,7 +84,7 @@ func (s *SyncService) SyncTrades(ctx context.Context, exchange types.Exchange, s var lastID int64 = 0 if lastTrade != nil { lastID = lastTrade.ID - startTime = lastTrade.Time + startTime = time.Time(lastTrade.Time) logrus.Infof("found last trade, start from lastID = %d since %s", lastID, startTime) } diff --git a/pkg/types/batch.go b/pkg/types/batch.go index bb5c00322..33d8cd1da 100644 --- a/pkg/types/batch.go +++ b/pkg/types/batch.go @@ -155,7 +155,7 @@ func (e ExchangeBatchProcessor) BatchQueryTrades(ctx context.Context, symbol str logrus.Infof("returned %d trades", len(trades)) - startTime = trades[len(trades)-1].Time + startTime = time.Time(trades[len(trades)-1].Time) for _, t := range trades { // ignore the first trade if last TradeID is given if t.ID == lastTradeID { diff --git a/pkg/types/trade.go b/pkg/types/trade.go index 56e82e409..1f2696975 100644 --- a/pkg/types/trade.go +++ b/pkg/types/trade.go @@ -3,10 +3,10 @@ package types import ( "fmt" "sync" - "time" "github.com/slack-go/slack" + "github.com/c9s/bbgo/pkg/datatype" "github.com/c9s/bbgo/pkg/util" ) @@ -49,12 +49,12 @@ type Trade struct { QuoteQuantity float64 `json:"quoteQuantity" db:"quote_quantity"` Symbol string `json:"symbol" db:"symbol"` - Side SideType `json:"side" db:"side"` - IsBuyer bool `json:"isBuyer" db:"is_buyer"` - IsMaker bool `json:"isMaker" db:"is_maker"` - Time time.Time `json:"tradedAt" db:"traded_at"` - Fee float64 `json:"fee" db:"fee"` - FeeCurrency string `json:"feeCurrency" db:"fee_currency"` + Side SideType `json:"side" db:"side"` + IsBuyer bool `json:"isBuyer" db:"is_buyer"` + IsMaker bool `json:"isMaker" db:"is_maker"` + Time datatype.Time `json:"tradedAt" db:"traded_at"` + Fee float64 `json:"fee" db:"fee"` + FeeCurrency string `json:"feeCurrency" db:"fee_currency"` IsMargin bool `json:"isMargin" db:"is_margin"` IsIsolated bool `json:"isIsolated" db:"is_isolated"` From ea58a3b092da5fc7742cb760b653b380b7a6d446 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 12:40:48 +0800 Subject: [PATCH 30/33] README.md: update readme for community and contributions --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 641f1eb6d..951680059 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,21 @@ You may register your exchange account with my referral ID to support this proje - BTC address `3J6XQJNWT56amqz9Hz2BEVQ7W4aNmb5kiU` - USDT ERC20 address `0x63E5805e027548A384c57E20141f6778591Bac6F` + +## Community + +You can join our telegram channel , it's in Chinese, but English is fine as well. + +## Contribution + +BBGO has a token BBG for the ecosystem (contract address: ). + +Each issue has its BBG label, by completing the issue with a pull request, you can get correspond amount of BBG. + +If you have feature request, you can offer your BBG for contributors. + +For further request, please contact us: + ## License MIT License From 20e6e4c299f4b5548313308758159f4e39fe622a Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 14:25:38 +0800 Subject: [PATCH 31/33] add MarshalJSON and UnmarshalJSON to datatype.Time --- pkg/datatype/time.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/datatype/time.go b/pkg/datatype/time.go index e871c2273..1365c5cea 100644 --- a/pkg/datatype/time.go +++ b/pkg/datatype/time.go @@ -10,6 +10,14 @@ type Time time.Time var layout = "2006-01-02 15:04:05.999Z07:00" +func (t *Time) UnmarshalJSON(data []byte) error { + return (*time.Time)(t).UnmarshalJSON(data) +} + +func (t Time) MarshalJSON() ([]byte, error) { + return time.Time(t).MarshalJSON() +} + func (t Time) String() string { return time.Time(t).String() } From 26f9e5488da81629e091f0f61a4fc30a385cba31 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 14:30:00 +0800 Subject: [PATCH 32/33] apply datatype.Time to order time fields --- pkg/backtest/matching.go | 4 ++-- pkg/datatype/time.go | 4 ++++ pkg/exchange/binance/convert.go | 4 ++-- pkg/exchange/binance/parse.go | 2 +- pkg/exchange/max/convert.go | 4 ++-- pkg/exchange/max/stream.go | 2 +- pkg/service/sync.go | 2 +- pkg/types/batch.go | 2 +- pkg/types/order.go | 18 +++++++++--------- 9 files changed, 23 insertions(+), 19 deletions(-) diff --git a/pkg/backtest/matching.go b/pkg/backtest/matching.go index e54bc06a2..fb963d806 100644 --- a/pkg/backtest/matching.go +++ b/pkg/backtest/matching.go @@ -437,7 +437,7 @@ func (m *SimplePriceMatching) newOrder(o types.SubmitOrder, orderID uint64) type Status: types.OrderStatusNew, ExecutedQuantity: 0, IsWorking: true, - CreationTime: m.CurrentTime, - UpdateTime: m.CurrentTime, + CreationTime: datatype.Time(m.CurrentTime), + UpdateTime: datatype.Time(m.CurrentTime), } } diff --git a/pkg/datatype/time.go b/pkg/datatype/time.go index 1365c5cea..10e27c594 100644 --- a/pkg/datatype/time.go +++ b/pkg/datatype/time.go @@ -22,6 +22,10 @@ func (t Time) String() string { return time.Time(t).String() } +func (t Time) Time() time.Time { + return time.Time(t) +} + // driver.Valuer interface // see http://jmoiron.net/blog/built-in-interfaces/ func (t Time) Value() (driver.Value, error) { diff --git a/pkg/exchange/binance/convert.go b/pkg/exchange/binance/convert.go index a1185f4b0..01dbee100 100644 --- a/pkg/exchange/binance/convert.go +++ b/pkg/exchange/binance/convert.go @@ -60,8 +60,8 @@ func ToGlobalOrder(binanceOrder *binance.Order, isMargin bool) (*types.Order, er OrderID: uint64(binanceOrder.OrderID), Status: toGlobalOrderStatus(binanceOrder.Status), ExecutedQuantity: util.MustParseFloat(binanceOrder.ExecutedQuantity), - CreationTime: millisecondTime(binanceOrder.Time), - UpdateTime: millisecondTime(binanceOrder.UpdateTime), + CreationTime: datatype.Time(millisecondTime(binanceOrder.Time)), + UpdateTime: datatype.Time(millisecondTime(binanceOrder.UpdateTime)), IsMargin: isMargin, IsIsolated: binanceOrder.IsIsolated, }, nil diff --git a/pkg/exchange/binance/parse.go b/pkg/exchange/binance/parse.go index d983ca69c..bfb6ebb3c 100644 --- a/pkg/exchange/binance/parse.go +++ b/pkg/exchange/binance/parse.go @@ -113,7 +113,7 @@ func (e *ExecutionReportEvent) Order() (*types.Order, error) { OrderID: uint64(e.OrderID), Status: toGlobalOrderStatus(binance.OrderStatusType(e.CurrentOrderStatus)), ExecutedQuantity: util.MustParseFloat(e.CumulativeFilledQuantity), - CreationTime: orderCreationTime, + CreationTime: datatype.Time(orderCreationTime), }, nil } diff --git a/pkg/exchange/max/convert.go b/pkg/exchange/max/convert.go index 0586f9e5b..2940d55fa 100644 --- a/pkg/exchange/max/convert.go +++ b/pkg/exchange/max/convert.go @@ -167,8 +167,8 @@ func toGlobalOrder(maxOrder max.Order) (*types.Order, error) { OrderID: maxOrder.ID, Status: toGlobalOrderStatus(maxOrder.State, executedVolume, remainingVolume), ExecutedQuantity: executedVolume.Float64(), - CreationTime: maxOrder.CreatedAt, - UpdateTime: maxOrder.CreatedAt, + CreationTime: datatype.Time(maxOrder.CreatedAt), + UpdateTime: datatype.Time(maxOrder.CreatedAt), }, nil } diff --git a/pkg/exchange/max/stream.go b/pkg/exchange/max/stream.go index 67f6cc521..fd0f557bd 100644 --- a/pkg/exchange/max/stream.go +++ b/pkg/exchange/max/stream.go @@ -229,6 +229,6 @@ func toGlobalOrderUpdate(u max.OrderUpdate) (*types.Order, error) { OrderID: u.ID, Status: toGlobalOrderStatus(u.State, executedVolume, remainingVolume), ExecutedQuantity: executedVolume.Float64(), - CreationTime: time.Unix(0, u.CreatedAtMs*int64(time.Millisecond)), + CreationTime: datatype.Time(time.Unix(0, u.CreatedAtMs*int64(time.Millisecond))), }, nil } diff --git a/pkg/service/sync.go b/pkg/service/sync.go index a1d58e629..ef2e4ed52 100644 --- a/pkg/service/sync.go +++ b/pkg/service/sync.go @@ -34,7 +34,7 @@ func (s *SyncService) SyncOrders(ctx context.Context, exchange types.Exchange, s var lastID uint64 = 0 if lastOrder != nil { lastID = lastOrder.OrderID - startTime = lastOrder.CreationTime + startTime = lastOrder.CreationTime.Time() logrus.Infof("found last order, start from lastID = %d since %s", lastID, startTime) } diff --git a/pkg/types/batch.go b/pkg/types/batch.go index 33d8cd1da..dee297509 100644 --- a/pkg/types/batch.go +++ b/pkg/types/batch.go @@ -51,7 +51,7 @@ func (e ExchangeBatchProcessor) BatchQueryClosedOrders(ctx context.Context, symb } c <- o - startTime = o.CreationTime + startTime = o.CreationTime.Time() lastOrderID = o.OrderID orderIDs[o.OrderID] = struct{}{} } diff --git a/pkg/types/order.go b/pkg/types/order.go index 3d7c4bfee..0a35744cc 100644 --- a/pkg/types/order.go +++ b/pkg/types/order.go @@ -2,10 +2,10 @@ package types import ( "fmt" - "time" "github.com/slack-go/slack" + "github.com/c9s/bbgo/pkg/datatype" "github.com/c9s/bbgo/pkg/util" ) @@ -113,14 +113,14 @@ func (o *SubmitOrder) SlackAttachment() slack.Attachment { type Order struct { SubmitOrder - Exchange string `json:"exchange" db:"exchange"` - GID uint64 `json:"gid" db:"gid"` - OrderID uint64 `json:"orderID" db:"order_id"` // order id - Status OrderStatus `json:"status" db:"status"` - ExecutedQuantity float64 `json:"executedQuantity" db:"executed_quantity"` - IsWorking bool `json:"isWorking" db:"is_working"` - CreationTime time.Time `json:"creationTime" db:"created_at"` - UpdateTime time.Time `json:"updateTime" db:"updated_at"` + Exchange string `json:"exchange" db:"exchange"` + GID uint64 `json:"gid" db:"gid"` + OrderID uint64 `json:"orderID" db:"order_id"` // order id + Status OrderStatus `json:"status" db:"status"` + ExecutedQuantity float64 `json:"executedQuantity" db:"executed_quantity"` + IsWorking bool `json:"isWorking" db:"is_working"` + CreationTime datatype.Time `json:"creationTime" db:"created_at"` + UpdateTime datatype.Time `json:"updateTime" db:"updated_at"` IsMargin bool `json:"isMargin" db:"is_margin"` IsIsolated bool `json:"isIsolated" db:"is_isolated"` From 855378e0980179653a1719ce64815e8ffb52add2 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 14:31:46 +0800 Subject: [PATCH 33/33] add driver field to the setup db route --- pkg/server/setup.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/server/setup.go b/pkg/server/setup.go index d81a7f9d4..9e5e482b1 100644 --- a/pkg/server/setup.go +++ b/pkg/server/setup.go @@ -45,7 +45,8 @@ func (s *Server) setupTestDB(c *gin.Context) { func (s *Server) setupConfigureDB(c *gin.Context) { payload := struct { - DSN string `json:"dsn"` + Driver string `json:"driver"` + DSN string `json:"dsn"` }{} if err := c.BindJSON(&payload); err != nil { @@ -53,19 +54,25 @@ func (s *Server) setupConfigureDB(c *gin.Context) { return } - dsn := payload.DSN - if len(dsn) == 0 { + if len(payload.DSN) == 0 { c.JSON(http.StatusBadRequest, gin.H{"error": "missing dsn argument"}) return } - driver := "mysql" - if err := s.Environ.ConfigureDatabase(c, driver, dsn); err != nil { + if payload.Driver == "" { + payload.Driver = "mysql" + } + + if err := s.Environ.ConfigureDatabase(c, payload.Driver, payload.DSN); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - c.JSON(http.StatusOK, gin.H{"success": true}) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "driver": payload.Driver, + "dsn": payload.DSN, + }) } func (s *Server) setupAddStrategy(c *gin.Context) {