From ea0c20cfe7d240dc9d55c8eabfee3fef2cb6ff54 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 13:04:52 +0800 Subject: [PATCH 01/23] rename enableApiServer to enableWebServer --- pkg/cmd/run.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index ccb0e7415..b246d2d59 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -331,7 +331,7 @@ func run(cmd *cobra.Command, args []string) error { return err } - enableApiServer, err := cmd.Flags().GetBool("enable-web-server") + enableWebServer, err := cmd.Flags().GetBool("enable-web-server") if err != nil { return err } @@ -348,10 +348,7 @@ func run(cmd *cobra.Command, args []string) error { var userConfig = &bbgo.Config{} - if setup { - log.Infof("running in setup mode, skip reading config file") - enableApiServer = true - } else { + if !setup { // if it's not setup, then the config file option is required. if len(configFile) == 0 { return errors.New("--config option is required") @@ -377,7 +374,7 @@ func run(cmd *cobra.Command, args []string) error { } if setup { - return runSetup(ctx, userConfig, enableApiServer) + return runSetup(ctx, userConfig, true) } userConfig, err = bbgo.Load(configFile, true) @@ -385,7 +382,7 @@ func run(cmd *cobra.Command, args []string) error { return err } - return runConfig(ctx, userConfig, enableApiServer) + return runConfig(ctx, userConfig, enableWebServer) } return runWrapperBinary(ctx, userConfig, cmd, args) From 8cb77075d03e7e9bb4a16f00f650f44e6898dfe4 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 13:26:41 +0800 Subject: [PATCH 02/23] .travis.yml: get rockhopper before we test --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c36fe7efb..ac48e49fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ go: services: - redis-server before_script: +- go get github.com/c9s/rockhopper - go mod download script: - go test -v ./pkg/... From 007b67976829a53500bd97567eee167d542bfdb5 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 14:15:28 +0800 Subject: [PATCH 03/23] add sqlite3 migrations --- migrations/20200721225616_trades.sql | 3 +- migrations/sqlite3/20200721225616_trades.sql | 20 +++++++++ .../sqlite3/20200819054742_trade_index.sql | 9 ++++ migrations/sqlite3/20201102222546_orders.sql | 25 +++++++++++ .../20201103173342_trades_add_order_id.sql | 5 +++ .../20201105092857_trades_index_fix.sql | 19 ++++++++ .../20201105093056_orders_add_index.sql | 7 +++ migrations/sqlite3/20201106114742_klines.sql | 44 +++++++++++++++++++ .../20201211175751_fix_symbol_length.sql | 5 +++ .../20210118163847_fix_unique_index.sql | 9 ++++ .../20210119232826_add_margin_columns.sql | 33 ++++++++++++++ ...10129182704_trade_price_quantity_index.sql | 10 +++++ 12 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 migrations/sqlite3/20200721225616_trades.sql create mode 100644 migrations/sqlite3/20200819054742_trade_index.sql create mode 100644 migrations/sqlite3/20201102222546_orders.sql create mode 100644 migrations/sqlite3/20201103173342_trades_add_order_id.sql create mode 100644 migrations/sqlite3/20201105092857_trades_index_fix.sql create mode 100644 migrations/sqlite3/20201105093056_orders_add_index.sql create mode 100644 migrations/sqlite3/20201106114742_klines.sql create mode 100644 migrations/sqlite3/20201211175751_fix_symbol_length.sql create mode 100644 migrations/sqlite3/20210118163847_fix_unique_index.sql create mode 100644 migrations/sqlite3/20210119232826_add_margin_columns.sql create mode 100644 migrations/sqlite3/20210129182704_trade_price_quantity_index.sql diff --git a/migrations/20200721225616_trades.sql b/migrations/20200721225616_trades.sql index 12d4e4c77..27760ac23 100644 --- a/migrations/20200721225616_trades.sql +++ b/migrations/20200721225616_trades.sql @@ -19,5 +19,6 @@ CREATE TABLE `trades` PRIMARY KEY (`gid`), UNIQUE KEY `id` (`id`) ); + -- +down -DROP TABLE `trades`; +DROP TABLE IF EXISTS `trades`; diff --git a/migrations/sqlite3/20200721225616_trades.sql b/migrations/sqlite3/20200721225616_trades.sql new file mode 100644 index 000000000..fcd5f8f50 --- /dev/null +++ b/migrations/sqlite3/20200721225616_trades.sql @@ -0,0 +1,20 @@ +-- +up +CREATE TABLE `trades` +( + `gid` INTEGER PRIMARY KEY AUTOINCREMENT, + `id` INTEGER, + `exchange` TEXT NOT NULL DEFAULT '', + `symbol` TEXT NOT NULL, + `price` DECIMAL(16, 8) NOT NULL, + `quantity` DECIMAL(16, 8) NOT NULL, + `quote_quantity` DECIMAL(16, 8) NOT NULL, + `fee` DECIMAL(16, 8) NOT NULL, + `fee_currency` VARCHAR(4) NOT NULL, + `is_buyer` BOOLEAN NOT NULL DEFAULT FALSE, + `is_maker` BOOLEAN NOT NULL DEFAULT FALSE, + `side` VARCHAR(4) NOT NULL DEFAULT '', + `traded_at` DATETIME(3) NOT NULL +); + +-- +down +DROP TABLE IF EXISTS `trades`; diff --git a/migrations/sqlite3/20200819054742_trade_index.sql b/migrations/sqlite3/20200819054742_trade_index.sql new file mode 100644 index 000000000..6ce90b1bd --- /dev/null +++ b/migrations/sqlite3/20200819054742_trade_index.sql @@ -0,0 +1,9 @@ +-- +up +CREATE INDEX trades_symbol ON trades(symbol); +CREATE INDEX trades_symbol_fee_currency ON trades(symbol, fee_currency, traded_at); +CREATE INDEX trades_traded_at_symbol ON trades(traded_at, symbol); + +-- +down +DROP INDEX trades_symbol ON trades; +DROP INDEX trades_symbol_fee_currency ON trades; +DROP INDEX trades_traded_at_symbol ON trades; diff --git a/migrations/sqlite3/20201102222546_orders.sql b/migrations/sqlite3/20201102222546_orders.sql new file mode 100644 index 000000000..561afd06a --- /dev/null +++ b/migrations/sqlite3/20201102222546_orders.sql @@ -0,0 +1,25 @@ +-- +up +CREATE TABLE `orders` +( + `gid` INTEGER PRIMARY KEY AUTOINCREMENT, + + `exchange` VARCHAR NOT NULL DEFAULT '', + -- order_id is the order id returned from the exchange + `order_id` INTEGER NOT NULL, + `client_order_id` VARCHAR NOT NULL DEFAULT '', + `order_type` VARCHAR NOT NULL, + `symbol` VARCHAR NOT NULL, + `status` VARCHAR NOT NULL, + `time_in_force` VARCHAR NOT NULL, + `price` DECIMAL(16, 8) NOT NULL, + `stop_price` DECIMAL(16, 8) NOT NULL, + `quantity` DECIMAL(16, 8) NOT NULL, + `executed_quantity` DECIMAL(16, 8) NOT NULL DEFAULT 0.0, + `side` VARCHAR NOT NULL DEFAULT '', + `is_working` BOOLEAN NOT NULL DEFAULT FALSE, + `created_at` DATETIME(3) NOT NULL, + `updated_at` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- +down +DROP TABLE IF EXISTS `orders`; diff --git a/migrations/sqlite3/20201103173342_trades_add_order_id.sql b/migrations/sqlite3/20201103173342_trades_add_order_id.sql new file mode 100644 index 000000000..7ff284200 --- /dev/null +++ b/migrations/sqlite3/20201103173342_trades_add_order_id.sql @@ -0,0 +1,5 @@ +-- +up +ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER NOT NULL; + +-- +down +ALTER TABLE `trades` RENAME COLUMN `order_id` TO `order_id_deleted`; diff --git a/migrations/sqlite3/20201105092857_trades_index_fix.sql b/migrations/sqlite3/20201105092857_trades_index_fix.sql new file mode 100644 index 000000000..44f58e768 --- /dev/null +++ b/migrations/sqlite3/20201105092857_trades_index_fix.sql @@ -0,0 +1,19 @@ +-- +up +DROP INDEX IF EXISTS trades_symbol; +DROP INDEX IF EXISTS trades_symbol_fee_currency; +DROP INDEX IF EXISTS trades_traded_at_symbol; + +CREATE INDEX trades_symbol ON trades (exchange, symbol); +CREATE INDEX trades_symbol_fee_currency ON trades (exchange, symbol, fee_currency, traded_at); +CREATE INDEX trades_traded_at_symbol ON trades (exchange, traded_at, symbol); + +-- +down +DROP INDEX IF EXISTS trades_symbol ON trades; +DROP INDEX IF EXISTS trades_symbol_fee_currency ON trades; +DROP INDEX IF EXISTS trades_traded_at_symbol ON trades; + +CREATE INDEX trades_symbol ON trades (symbol); +CREATE INDEX trades_symbol_fee_currency ON trades (symbol, fee_currency, traded_at); +CREATE INDEX trades_traded_at_symbol ON trades (traded_at, symbol); + + diff --git a/migrations/sqlite3/20201105093056_orders_add_index.sql b/migrations/sqlite3/20201105093056_orders_add_index.sql new file mode 100644 index 000000000..99834a551 --- /dev/null +++ b/migrations/sqlite3/20201105093056_orders_add_index.sql @@ -0,0 +1,7 @@ +-- +up +CREATE INDEX orders_symbol ON orders (exchange, symbol); +CREATE UNIQUE INDEX orders_order_id ON orders (order_id, exchange); + +-- +down +DROP INDEX IF EXISTS orders_symbol; +DROP INDEX IF EXISTS orders_order_id; diff --git a/migrations/sqlite3/20201106114742_klines.sql b/migrations/sqlite3/20201106114742_klines.sql new file mode 100644 index 000000000..72c679206 --- /dev/null +++ b/migrations/sqlite3/20201106114742_klines.sql @@ -0,0 +1,44 @@ +-- +up +-- +begin +CREATE TABLE `klines` +( + `gid` INTEGER PRIMARY KEY AUTOINCREMENT, + `exchange` VARCHAR(10) NOT NULL, + `start_time` DATETIME(3) NOT NULL, + `end_time` DATETIME(3) NOT NULL, + `interval` VARCHAR(3) NOT NULL, + `symbol` VARCHAR(7) NOT NULL, + `open` DECIMAL(16, 8) NOT NULL, + `high` DECIMAL(16, 8) NOT NULL, + `low` DECIMAL(16, 8) NOT NULL, + `close` DECIMAL(16, 8) NOT NULL DEFAULT 0.0, + `volume` DECIMAL(16, 8) NOT NULL DEFAULT 0.0, + `closed` BOOLEAN NOT NULL DEFAULT TRUE, + `last_trade_id` INT NOT NULL DEFAULT 0, + `num_trades` INT NOT NULL DEFAULT 0 +); +-- +end + +-- +begin +CREATE INDEX `klines_end_time_symbol_interval` ON klines (`end_time`, `symbol`, `interval`); +-- +end + +-- +begin +CREATE TABLE `okex_klines` AS SELECT * FROM `klines` WHERE 0 +-- +end + +-- +begin +CREATE TABLE `binance_klines` AS SELECT * FROM `klines` WHERE 0 +-- +end + +-- +begin +CREATE TABLE `max_klines` AS SELECT * FROM `klines` WHERE 0 +-- +end + +-- +down +DROP INDEX IF EXISTS `klines_end_time_symbol_interval`; +DROP TABLE IF EXISTS `binance_klines`; +DROP TABLE IF EXISTS `okex_klines`; +DROP TABLE IF EXISTS `max_klines`; +DROP TABLE IF EXISTS `klines`; + diff --git a/migrations/sqlite3/20201211175751_fix_symbol_length.sql b/migrations/sqlite3/20201211175751_fix_symbol_length.sql new file mode 100644 index 000000000..06569c667 --- /dev/null +++ b/migrations/sqlite3/20201211175751_fix_symbol_length.sql @@ -0,0 +1,5 @@ +-- +up +SELECT 1; + +-- +down +SELECT 1; diff --git a/migrations/sqlite3/20210118163847_fix_unique_index.sql b/migrations/sqlite3/20210118163847_fix_unique_index.sql new file mode 100644 index 000000000..60ada793e --- /dev/null +++ b/migrations/sqlite3/20210118163847_fix_unique_index.sql @@ -0,0 +1,9 @@ +-- +up +-- +begin +CREATE UNIQUE INDEX `trade_unique_id` ON `trades` (`exchange`,`symbol`, `side`, `id`); +-- +end + +-- +down +-- +begin +DROP INDEX IF EXISTS `trade_unique_id`; +-- +end diff --git a/migrations/sqlite3/20210119232826_add_margin_columns.sql b/migrations/sqlite3/20210119232826_add_margin_columns.sql new file mode 100644 index 000000000..eaff7b613 --- /dev/null +++ b/migrations/sqlite3/20210119232826_add_margin_columns.sql @@ -0,0 +1,33 @@ +-- +up +-- +begin +ALTER TABLE `trades` ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end +-- +begin +ALTER TABLE `trades` ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end + +-- +begin +ALTER TABLE `orders` ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end + +-- +begin +ALTER TABLE `orders` ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE; +-- +end + +-- +down + +-- +begin +ALTER TABLE `trades` RENAME COLUMN `is_margin` TO `is_margin_deleted`; +-- +end + +-- +begin +ALTER TABLE `trades` RENAME COLUMN `is_isolated` TO `is_isolated_deleted`; +-- +end + +-- +begin +ALTER TABLE `orders` RENAME COLUMN `is_margin` TO `is_margin_deleted`; +-- +end + +-- +begin +ALTER TABLE `orders` RENAME COLUMN `is_isolated` TO `is_isolated_deleted`; +-- +end diff --git a/migrations/sqlite3/20210129182704_trade_price_quantity_index.sql b/migrations/sqlite3/20210129182704_trade_price_quantity_index.sql new file mode 100644 index 000000000..196f7467d --- /dev/null +++ b/migrations/sqlite3/20210129182704_trade_price_quantity_index.sql @@ -0,0 +1,10 @@ +-- +up +-- +begin +CREATE INDEX trades_price_quantity ON trades (order_id,price,quantity); +-- +end + +-- +down + +-- +begin +DROP INDEX trades_price_quantity; +-- +end From 16b7944b925725fe838f9c77d4f0449cff8eb553 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 14:15:44 +0800 Subject: [PATCH 04/23] add rockhopper config for sqlite3 --- rockhopper_sqlite.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 rockhopper_sqlite.yaml diff --git a/rockhopper_sqlite.yaml b/rockhopper_sqlite.yaml new file mode 100644 index 000000000..259e7fe67 --- /dev/null +++ b/rockhopper_sqlite.yaml @@ -0,0 +1,5 @@ +--- +driver: sqlite3 +dialect: sqlite3 +dsn: "bbgo.sqlite3" +migrationsDir: migrations/sqlite3 From ccb1708fd96fa813ba96a97e62818918fde1068b Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 14:20:07 +0800 Subject: [PATCH 05/23] add script for testing sqlite3 migration --- .travis.yml | 1 + migrations/sqlite3/20200819054742_trade_index.sql | 6 +++--- migrations/sqlite3/20201105092857_trades_index_fix.sql | 6 +++--- scripts/test-sqlite3-migrations.sh | 2 ++ 4 files changed, 9 insertions(+), 6 deletions(-) create mode 100755 scripts/test-sqlite3-migrations.sh diff --git a/.travis.yml b/.travis.yml index ac48e49fe..595d79af6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,4 +9,5 @@ before_script: - go get github.com/c9s/rockhopper - go mod download script: +- bash scripts/test-sqlite3-migrations.sh - go test -v ./pkg/... diff --git a/migrations/sqlite3/20200819054742_trade_index.sql b/migrations/sqlite3/20200819054742_trade_index.sql index 6ce90b1bd..e6dba452b 100644 --- a/migrations/sqlite3/20200819054742_trade_index.sql +++ b/migrations/sqlite3/20200819054742_trade_index.sql @@ -4,6 +4,6 @@ CREATE INDEX trades_symbol_fee_currency ON trades(symbol, fee_currency, traded_a CREATE INDEX trades_traded_at_symbol ON trades(traded_at, symbol); -- +down -DROP INDEX trades_symbol ON trades; -DROP INDEX trades_symbol_fee_currency ON trades; -DROP INDEX trades_traded_at_symbol ON trades; +DROP INDEX trades_symbol; +DROP INDEX trades_symbol_fee_currency; +DROP INDEX trades_traded_at_symbol; diff --git a/migrations/sqlite3/20201105092857_trades_index_fix.sql b/migrations/sqlite3/20201105092857_trades_index_fix.sql index 44f58e768..30a955406 100644 --- a/migrations/sqlite3/20201105092857_trades_index_fix.sql +++ b/migrations/sqlite3/20201105092857_trades_index_fix.sql @@ -8,9 +8,9 @@ CREATE INDEX trades_symbol_fee_currency ON trades (exchange, symbol, fee_currenc CREATE INDEX trades_traded_at_symbol ON trades (exchange, traded_at, symbol); -- +down -DROP INDEX IF EXISTS trades_symbol ON trades; -DROP INDEX IF EXISTS trades_symbol_fee_currency ON trades; -DROP INDEX IF EXISTS trades_traded_at_symbol ON trades; +DROP INDEX IF EXISTS trades_symbol; +DROP INDEX IF EXISTS trades_symbol_fee_currency; +DROP INDEX IF EXISTS trades_traded_at_symbol; CREATE INDEX trades_symbol ON trades (symbol); CREATE INDEX trades_symbol_fee_currency ON trades (symbol, fee_currency, traded_at); diff --git a/scripts/test-sqlite3-migrations.sh b/scripts/test-sqlite3-migrations.sh new file mode 100755 index 000000000..83c5b683c --- /dev/null +++ b/scripts/test-sqlite3-migrations.sh @@ -0,0 +1,2 @@ +#!/bin/bash +rm -v bbgo.sqlite3 && rockhopper --config rockhopper_sqlite.yaml up && rockhopper --config rockhopper_sqlite.yaml down --to 1 From 01ea0885dcffbd0a60fedf25ec1620fb9c04f1c8 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 14:33:12 +0800 Subject: [PATCH 06/23] scripts: use force flag to remove bbgo.sqlite3 --- scripts/test-sqlite3-migrations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-sqlite3-migrations.sh b/scripts/test-sqlite3-migrations.sh index 83c5b683c..c552aed86 100755 --- a/scripts/test-sqlite3-migrations.sh +++ b/scripts/test-sqlite3-migrations.sh @@ -1,2 +1,2 @@ #!/bin/bash -rm -v bbgo.sqlite3 && rockhopper --config rockhopper_sqlite.yaml up && rockhopper --config rockhopper_sqlite.yaml down --to 1 +rm -fv bbgo.sqlite3 && rockhopper --config rockhopper_sqlite.yaml up && rockhopper --config rockhopper_sqlite.yaml down --to 1 From 483f6270f40f38e2e3dd7cbccc4ad2fcd5a78afb Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 5 Feb 2021 14:44:29 +0800 Subject: [PATCH 07/23] .travis.yml: add install section --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 595d79af6..4123aafac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,11 @@ go: - 1.15 services: - redis-server + +install: +- go get github.com/c9s/rockhopper/cmd/rockhopper + before_script: -- go get github.com/c9s/rockhopper - go mod download script: - bash scripts/test-sqlite3-migrations.sh From 256fce484c430b195bb74bbd5062b61a1915d2e5 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:15:03 +0800 Subject: [PATCH 08/23] add default 0 to not null field --- migrations/sqlite3/20201103173342_trades_add_order_id.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/sqlite3/20201103173342_trades_add_order_id.sql b/migrations/sqlite3/20201103173342_trades_add_order_id.sql index 7ff284200..8f0f94675 100644 --- a/migrations/sqlite3/20201103173342_trades_add_order_id.sql +++ b/migrations/sqlite3/20201103173342_trades_add_order_id.sql @@ -1,5 +1,5 @@ -- +up -ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER NOT NULL; +ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER NOT NULL DEFATUL 0; -- +down ALTER TABLE `trades` RENAME COLUMN `order_id` TO `order_id_deleted`; From 2691d06f5f3e8688129085d7ed68b14abed54e74 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:15:25 +0800 Subject: [PATCH 09/23] add rockhopper config for mysql --- rockhopper_mysql.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 rockhopper_mysql.yaml diff --git a/rockhopper_mysql.yaml b/rockhopper_mysql.yaml new file mode 100644 index 000000000..01e86e7bf --- /dev/null +++ b/rockhopper_mysql.yaml @@ -0,0 +1,5 @@ +--- +driver: mysql +dialect: mysql +dsn: "root@tcp(localhost:3306)/bbgo?parseTime=true" +migrationsDir: migrations From 6d756d2409644f62ae4ee17d5addc2baad383ce7 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:16:04 +0800 Subject: [PATCH 10/23] .travis.yml: add mysql to travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4123aafac..7d8e3dc53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,10 @@ language: go go: - 1.14 - 1.15 + services: - redis-server +- mysql install: - go get github.com/c9s/rockhopper/cmd/rockhopper From 94dc291580502da90064aa8c1f768fa918e78da6 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:16:56 +0800 Subject: [PATCH 11/23] .travis.yml: create bbgo mysql database --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7d8e3dc53..3d513d589 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,15 @@ services: - redis-server - mysql +before_install: +- mysql -e 'CREATE DATABASE bbgo;' + install: - go get github.com/c9s/rockhopper/cmd/rockhopper before_script: - go mod download + script: - bash scripts/test-sqlite3-migrations.sh - go test -v ./pkg/... From de51eb29e481988c0893f410e15a1166caf4c95a Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:48:05 +0800 Subject: [PATCH 12/23] refactor db stuff with database service --- pkg/bbgo/environment.go | 22 +++++------- pkg/cmd/backtest.go | 2 +- pkg/cmd/cancel.go | 7 ++-- pkg/cmd/run.go | 2 +- pkg/cmd/sync.go | 2 +- pkg/server/routes.go | 5 +-- pkg/server/setup.go | 2 +- pkg/service/database.go | 80 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 98 insertions(+), 24 deletions(-) create mode 100644 pkg/service/database.go diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index 11780b9ae..c03c8a711 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -7,7 +7,6 @@ import ( "time" "github.com/codingconcepts/env" - "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -48,6 +47,7 @@ type Environment struct { PersistenceServiceFacade *PersistenceServiceFacade + DatabaseService *service.DatabaseService OrderService *service.OrderService TradeService *service.TradeService TradeSync *service.SyncService @@ -56,8 +56,6 @@ type Environment struct { startTime time.Time tradeScanTime time.Time sessions map[string]*ExchangeSession - - MysqlURL string } func NewEnvironment() *Environment { @@ -78,23 +76,19 @@ func (environ *Environment) Sessions() map[string]*ExchangeSession { return environ.sessions } -func (environ *Environment) ConfigureDatabase(ctx context.Context, dsn string) error { - db, err := ConnectMySQL(dsn) +func (environ *Environment) ConfigureDatabase(ctx context.Context, driver string, dsn string) error { + environ.DatabaseService = service.NewDatabaseService(driver, dsn) + err := environ.DatabaseService.Connect() if err != nil { return err } - environ.MysqlURL = dsn - - if err := upgradeDB(ctx, "mysql", db.DB); err != nil { + if err := environ.DatabaseService.Upgrade(ctx) ; err != nil { return err } - environ.SetDB(db) - return nil -} - -func (environ *Environment) SetDB(db *sqlx.DB) *Environment { + // get the db connection pool object to create other services + db := environ.DatabaseService.DB environ.OrderService = &service.OrderService{DB: db} environ.TradeService = &service.TradeService{DB: db} environ.TradeSync = &service.SyncService{ @@ -102,7 +96,7 @@ func (environ *Environment) SetDB(db *sqlx.DB) *Environment { OrderService: environ.OrderService, } - return environ + return nil } // AddExchangeSession adds the existing exchange session or pre-created exchange session diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 0402c561b..638a4885d 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -118,7 +118,7 @@ var BacktestCmd = &cobra.Command{ environ := bbgo.NewEnvironment() if viper.IsSet("mysql-url") { dsn := viper.GetString("mysql-url") - if err := environ.ConfigureDatabase(ctx, dsn); err != nil { + if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { return err } } diff --git a/pkg/cmd/cancel.go b/pkg/cmd/cancel.go index 03cc64a41..f64971d8e 100644 --- a/pkg/cmd/cancel.go +++ b/pkg/cmd/cancel.go @@ -3,11 +3,11 @@ package cmd import ( "context" "fmt" + "os" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/types" @@ -64,12 +64,11 @@ var CancelCmd = &cobra.Command{ environ := bbgo.NewEnvironment() - if viper.IsSet("mysql-url") { - db, err := bbgo.ConnectMySQL(viper.GetString("mysql-url")) + if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { + err := environ.ConfigureDatabase(ctx, "mysql", dsn) if err != nil { return err } - environ.SetDB(db) } if err := environ.AddExchangesFromConfig(userConfig); err != nil { diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index b246d2d59..90eea1123 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -97,7 +97,7 @@ func runSetup(baseCtx context.Context, userConfig *bbgo.Config, enableApiServer func BootstrapEnvironment(ctx context.Context, environ *bbgo.Environment, userConfig *bbgo.Config) error { if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - if err := environ.ConfigureDatabase(ctx, dsn); err != nil { + if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { return err } } diff --git a/pkg/cmd/sync.go b/pkg/cmd/sync.go index e2d761fdf..8f0b0314f 100644 --- a/pkg/cmd/sync.go +++ b/pkg/cmd/sync.go @@ -55,7 +55,7 @@ var SyncCmd = &cobra.Command{ if viper.IsSet("mysql-url") { dsn := viper.GetString("mysql-url") - if err := environ.ConfigureDatabase(ctx, dsn); err != nil { + if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { return err } } diff --git a/pkg/server/routes.go b/pkg/server/routes.go index c70f62a46..5320b6ddc 100644 --- a/pkg/server/routes.go +++ b/pkg/server/routes.go @@ -421,8 +421,9 @@ func (s *Server) setupSaveConfig(c *gin.Context) { return } - if len(s.Environ.MysqlURL) > 0 { - envVars["MYSQL_URL"] = s.Environ.MysqlURL + if s.Environ.DatabaseService != nil { + envVars["DB_DRIVER"] = s.Environ.DatabaseService.Driver + envVars["DB_DSN"] = s.Environ.DatabaseService.DSN } dotenvFile := ".env.local" diff --git a/pkg/server/setup.go b/pkg/server/setup.go index 4232d812c..75d3a5788 100644 --- a/pkg/server/setup.go +++ b/pkg/server/setup.go @@ -58,7 +58,7 @@ func (s *Server) setupConfigureDB(c *gin.Context) { return } - if err := s.Environ.ConfigureDatabase(c, dsn); err != nil { + if err := s.Environ.ConfigureDatabase(c, "mysql", dsn); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } diff --git a/pkg/service/database.go b/pkg/service/database.go new file mode 100644 index 000000000..47d797b75 --- /dev/null +++ b/pkg/service/database.go @@ -0,0 +1,80 @@ +package service + +import ( + "context" + + "github.com/c9s/rockhopper" + "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" +) + +type DatabaseService struct { + Driver string + DSN string + DB *sqlx.DB +} + +func NewDatabaseService(driver, dsn string) *DatabaseService { + if driver == "mysql" { + var err error + dsn, err = ReformatMysqlDSN(dsn) + if err != nil { + // incorrect mysql dsn is logical exception + panic(err) + } + } + + return &DatabaseService{ + Driver: driver, + DSN: dsn, + } + +} + +func (s *DatabaseService) Connect() error { + var err error + s.DB, err = sqlx.Connect(s.Driver, s.DSN) + return err +} + +func (s *DatabaseService) Close() error { + return s.DB.Close() +} + +func (s *DatabaseService) Upgrade(ctx context.Context) error { + dialect, err := rockhopper.LoadDialect(s.Driver) + if err != nil { + return err + } + + loader := &rockhopper.GoMigrationLoader{} + migrations, err := loader.Load() + if err != nil { + return err + } + + // sqlx.DB is different from sql.DB + rh := rockhopper.New(s.Driver, dialect, s.DB.DB) + + currentVersion, err := rh.CurrentVersion() + if err != nil { + return err + } + + if err := rockhopper.Up(ctx, rh, migrations, currentVersion, 0); err != nil { + return err + } + + return nil +} + +func ReformatMysqlDSN(dsn string) (string, error) { + config, err := mysql.ParseDSN(dsn) + if err != nil { + return "", err + } + + config.ParseTime = true + dsn = config.FormatDSN() + return dsn, nil +} From 786757a686e78529a3e5ebe76e6918f1a4d9c896 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:49:05 +0800 Subject: [PATCH 13/23] migrations/sqlite3: fix sqlite3 alter syntax --- migrations/sqlite3/20201103173342_trades_add_order_id.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/sqlite3/20201103173342_trades_add_order_id.sql b/migrations/sqlite3/20201103173342_trades_add_order_id.sql index 8f0f94675..f337eb231 100644 --- a/migrations/sqlite3/20201103173342_trades_add_order_id.sql +++ b/migrations/sqlite3/20201103173342_trades_add_order_id.sql @@ -1,5 +1,5 @@ -- +up -ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER NOT NULL DEFATUL 0; +ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER; -- +down ALTER TABLE `trades` RENAME COLUMN `order_id` TO `order_id_deleted`; From 32f74d8f6f5733f6ecde749ca4d88c8248571155 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:50:02 +0800 Subject: [PATCH 14/23] move mysql migrations to migrations/mysql --- migrations/{ => mysql}/20200721225616_trades.sql | 0 migrations/{ => mysql}/20200819054742_trade_index.sql | 0 migrations/{ => mysql}/20201102222546_orders.sql | 0 migrations/{ => mysql}/20201103173342_trades_add_order_id.sql | 0 migrations/{ => mysql}/20201105092857_trades_index_fix.sql | 0 migrations/{ => mysql}/20201105093056_orders_add_index.sql | 0 migrations/{ => mysql}/20201106114742_klines.sql | 0 migrations/{ => mysql}/20201211175751_fix_symbol_length.sql | 0 migrations/{ => mysql}/20210118163847_fix_unique_index.sql | 0 migrations/{ => mysql}/20210119232826_add_margin_columns.sql | 0 .../{ => mysql}/20210129182704_trade_price_quantity_index.sql | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename migrations/{ => mysql}/20200721225616_trades.sql (100%) rename migrations/{ => mysql}/20200819054742_trade_index.sql (100%) rename migrations/{ => mysql}/20201102222546_orders.sql (100%) rename migrations/{ => mysql}/20201103173342_trades_add_order_id.sql (100%) rename migrations/{ => mysql}/20201105092857_trades_index_fix.sql (100%) rename migrations/{ => mysql}/20201105093056_orders_add_index.sql (100%) rename migrations/{ => mysql}/20201106114742_klines.sql (100%) rename migrations/{ => mysql}/20201211175751_fix_symbol_length.sql (100%) rename migrations/{ => mysql}/20210118163847_fix_unique_index.sql (100%) rename migrations/{ => mysql}/20210119232826_add_margin_columns.sql (100%) rename migrations/{ => mysql}/20210129182704_trade_price_quantity_index.sql (100%) diff --git a/migrations/20200721225616_trades.sql b/migrations/mysql/20200721225616_trades.sql similarity index 100% rename from migrations/20200721225616_trades.sql rename to migrations/mysql/20200721225616_trades.sql diff --git a/migrations/20200819054742_trade_index.sql b/migrations/mysql/20200819054742_trade_index.sql similarity index 100% rename from migrations/20200819054742_trade_index.sql rename to migrations/mysql/20200819054742_trade_index.sql diff --git a/migrations/20201102222546_orders.sql b/migrations/mysql/20201102222546_orders.sql similarity index 100% rename from migrations/20201102222546_orders.sql rename to migrations/mysql/20201102222546_orders.sql diff --git a/migrations/20201103173342_trades_add_order_id.sql b/migrations/mysql/20201103173342_trades_add_order_id.sql similarity index 100% rename from migrations/20201103173342_trades_add_order_id.sql rename to migrations/mysql/20201103173342_trades_add_order_id.sql diff --git a/migrations/20201105092857_trades_index_fix.sql b/migrations/mysql/20201105092857_trades_index_fix.sql similarity index 100% rename from migrations/20201105092857_trades_index_fix.sql rename to migrations/mysql/20201105092857_trades_index_fix.sql diff --git a/migrations/20201105093056_orders_add_index.sql b/migrations/mysql/20201105093056_orders_add_index.sql similarity index 100% rename from migrations/20201105093056_orders_add_index.sql rename to migrations/mysql/20201105093056_orders_add_index.sql diff --git a/migrations/20201106114742_klines.sql b/migrations/mysql/20201106114742_klines.sql similarity index 100% rename from migrations/20201106114742_klines.sql rename to migrations/mysql/20201106114742_klines.sql diff --git a/migrations/20201211175751_fix_symbol_length.sql b/migrations/mysql/20201211175751_fix_symbol_length.sql similarity index 100% rename from migrations/20201211175751_fix_symbol_length.sql rename to migrations/mysql/20201211175751_fix_symbol_length.sql diff --git a/migrations/20210118163847_fix_unique_index.sql b/migrations/mysql/20210118163847_fix_unique_index.sql similarity index 100% rename from migrations/20210118163847_fix_unique_index.sql rename to migrations/mysql/20210118163847_fix_unique_index.sql diff --git a/migrations/20210119232826_add_margin_columns.sql b/migrations/mysql/20210119232826_add_margin_columns.sql similarity index 100% rename from migrations/20210119232826_add_margin_columns.sql rename to migrations/mysql/20210119232826_add_margin_columns.sql diff --git a/migrations/20210129182704_trade_price_quantity_index.sql b/migrations/mysql/20210129182704_trade_price_quantity_index.sql similarity index 100% rename from migrations/20210129182704_trade_price_quantity_index.sql rename to migrations/mysql/20210129182704_trade_price_quantity_index.sql From c7440a3ea40bc8859e4686159251b13d5b5777ac Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:51:05 +0800 Subject: [PATCH 15/23] compile and update migration package --- pkg/migrations/mysql/20200721225616_trades.go | 33 ++++++++ .../mysql/20200819054742_trade_index.go | 53 ++++++++++++ pkg/migrations/mysql/20201102222546_orders.go | 33 ++++++++ .../20201103173342_trades_add_order_id.go | 33 ++++++++ .../mysql/20201105092857_trades_index_fix.go | 83 +++++++++++++++++++ .../mysql/20201105093056_orders_add_index.go | 43 ++++++++++ pkg/migrations/mysql/20201106114742_klines.go | 73 ++++++++++++++++ .../mysql/20201211175751_fix_symbol_length.go | 43 ++++++++++ .../mysql/20210118163847_fix_unique_index.go | 43 ++++++++++ .../20210119232826_add_margin_columns.go | 43 ++++++++++ ...210129182704_trade_price_quantity_index.go | 33 ++++++++ .../sqlite3/20200721225616_trades.go | 33 ++++++++ .../sqlite3/20200819054742_trade_index.go | 53 ++++++++++++ .../sqlite3/20201102222546_orders.go | 33 ++++++++ .../20201103173342_trades_add_order_id.go | 33 ++++++++ .../20201105092857_trades_index_fix.go | 83 +++++++++++++++++++ .../20201105093056_orders_add_index.go | 43 ++++++++++ .../sqlite3/20201106114742_klines.go | 73 ++++++++++++++++ .../20201211175751_fix_symbol_length.go | 33 ++++++++ .../20210118163847_fix_unique_index.go | 33 ++++++++ .../20210119232826_add_margin_columns.go | 63 ++++++++++++++ ...210129182704_trade_price_quantity_index.go | 33 ++++++++ 22 files changed, 1026 insertions(+) create mode 100644 pkg/migrations/mysql/20200721225616_trades.go create mode 100644 pkg/migrations/mysql/20200819054742_trade_index.go create mode 100644 pkg/migrations/mysql/20201102222546_orders.go create mode 100644 pkg/migrations/mysql/20201103173342_trades_add_order_id.go create mode 100644 pkg/migrations/mysql/20201105092857_trades_index_fix.go create mode 100644 pkg/migrations/mysql/20201105093056_orders_add_index.go create mode 100644 pkg/migrations/mysql/20201106114742_klines.go create mode 100644 pkg/migrations/mysql/20201211175751_fix_symbol_length.go create mode 100644 pkg/migrations/mysql/20210118163847_fix_unique_index.go create mode 100644 pkg/migrations/mysql/20210119232826_add_margin_columns.go create mode 100644 pkg/migrations/mysql/20210129182704_trade_price_quantity_index.go create mode 100644 pkg/migrations/sqlite3/20200721225616_trades.go create mode 100644 pkg/migrations/sqlite3/20200819054742_trade_index.go create mode 100644 pkg/migrations/sqlite3/20201102222546_orders.go create mode 100644 pkg/migrations/sqlite3/20201103173342_trades_add_order_id.go create mode 100644 pkg/migrations/sqlite3/20201105092857_trades_index_fix.go create mode 100644 pkg/migrations/sqlite3/20201105093056_orders_add_index.go create mode 100644 pkg/migrations/sqlite3/20201106114742_klines.go create mode 100644 pkg/migrations/sqlite3/20201211175751_fix_symbol_length.go create mode 100644 pkg/migrations/sqlite3/20210118163847_fix_unique_index.go create mode 100644 pkg/migrations/sqlite3/20210119232826_add_margin_columns.go create mode 100644 pkg/migrations/sqlite3/20210129182704_trade_price_quantity_index.go diff --git a/pkg/migrations/mysql/20200721225616_trades.go b/pkg/migrations/mysql/20200721225616_trades.go new file mode 100644 index 000000000..094b205ea --- /dev/null +++ b/pkg/migrations/mysql/20200721225616_trades.go @@ -0,0 +1,33 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTrades, downTrades) +} + +func upTrades(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `trades`\n(\n `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n `id` BIGINT UNSIGNED,\n `exchange` VARCHAR(24) NOT NULL DEFAULT '',\n `symbol` VARCHAR(8) NOT NULL,\n `price` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `quantity` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `quote_quantity` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `fee` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `fee_currency` VARCHAR(4) NOT NULL,\n `is_buyer` BOOLEAN NOT NULL DEFAULT FALSE,\n `is_maker` BOOLEAN NOT NULL DEFAULT FALSE,\n `side` VARCHAR(4) NOT NULL DEFAULT '',\n `traded_at` DATETIME(3) NOT NULL,\n PRIMARY KEY (`gid`),\n UNIQUE KEY `id` (`id`)\n);") + if err != nil { + return err + } + + return err +} + +func downTrades(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `trades`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20200819054742_trade_index.go b/pkg/migrations/mysql/20200819054742_trade_index.go new file mode 100644 index 000000000..e734dd88c --- /dev/null +++ b/pkg/migrations/mysql/20200819054742_trade_index.go @@ -0,0 +1,53 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradeIndex, downTradeIndex) +} + +func upTradeIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades(symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades(symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades(traded_at, symbol);") + if err != nil { + return err + } + + return err +} + +func downTradeIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol_fee_currency ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_traded_at_symbol ON trades;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201102222546_orders.go b/pkg/migrations/mysql/20201102222546_orders.go new file mode 100644 index 000000000..04aaa0653 --- /dev/null +++ b/pkg/migrations/mysql/20201102222546_orders.go @@ -0,0 +1,33 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upOrders, downOrders) +} + +func upOrders(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `orders`\n(\n `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n `exchange` VARCHAR(24) NOT NULL DEFAULT '',\n -- order_id is the order id returned from the exchange\n `order_id` BIGINT UNSIGNED NOT NULL,\n `client_order_id` VARCHAR(42) NOT NULL DEFAULT '',\n `order_type` VARCHAR(16) NOT NULL,\n `symbol` VARCHAR(8) NOT NULL,\n `status` VARCHAR(12) NOT NULL,\n `time_in_force` VARCHAR(4) NOT NULL,\n `price` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `stop_price` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `quantity` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `executed_quantity` DECIMAL(16, 8) UNSIGNED NOT NULL DEFAULT 0.0,\n `side` VARCHAR(4) NOT NULL DEFAULT '',\n `is_working` BOOL NOT NULL DEFAULT FALSE,\n `created_at` DATETIME(3) NOT NULL,\n `updated_at` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),\n PRIMARY KEY (`gid`)\n);") + if err != nil { + return err + } + + return err +} + +func downOrders(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE `orders`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201103173342_trades_add_order_id.go b/pkg/migrations/mysql/20201103173342_trades_add_order_id.go new file mode 100644 index 000000000..6b6dea539 --- /dev/null +++ b/pkg/migrations/mysql/20201103173342_trades_add_order_id.go @@ -0,0 +1,33 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradesAddOrderId, downTradesAddOrderId) +} + +func upTradesAddOrderId(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades`\n ADD COLUMN `order_id` BIGINT UNSIGNED NOT NULL;") + if err != nil { + return err + } + + return err +} + +func downTradesAddOrderId(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades`\n DROP COLUMN `order_id`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201105092857_trades_index_fix.go b/pkg/migrations/mysql/20201105092857_trades_index_fix.go new file mode 100644 index 000000000..d5a0917f1 --- /dev/null +++ b/pkg/migrations/mysql/20201105092857_trades_index_fix.go @@ -0,0 +1,83 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradesIndexFix, downTradesIndexFix) +} + +func upTradesIndexFix(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol_fee_currency ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_traded_at_symbol ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades (exchange, symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades (exchange, symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades (exchange, traded_at, symbol);") + if err != nil { + return err + } + + return err +} + +func downTradesIndexFix(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol_fee_currency ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_traded_at_symbol ON trades;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades (symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades (symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades (traded_at, symbol);") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201105093056_orders_add_index.go b/pkg/migrations/mysql/20201105093056_orders_add_index.go new file mode 100644 index 000000000..5e13521c5 --- /dev/null +++ b/pkg/migrations/mysql/20201105093056_orders_add_index.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upOrdersAddIndex, downOrdersAddIndex) +} + +func upOrdersAddIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX orders_symbol ON orders (exchange, symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE UNIQUE INDEX orders_order_id ON orders (order_id, exchange);") + if err != nil { + return err + } + + return err +} + +func downOrdersAddIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX orders_symbol ON orders;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX orders_order_id ON orders;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201106114742_klines.go b/pkg/migrations/mysql/20201106114742_klines.go new file mode 100644 index 000000000..0f5a8e756 --- /dev/null +++ b/pkg/migrations/mysql/20201106114742_klines.go @@ -0,0 +1,73 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upKlines, downKlines) +} + +func upKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `klines`\n(\n `gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n `exchange` VARCHAR(10) NOT NULL,\n `start_time` DATETIME(3) NOT NULL,\n `end_time` DATETIME(3) NOT NULL,\n `interval` VARCHAR(3) NOT NULL,\n `symbol` VARCHAR(7) NOT NULL,\n `open` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `high` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `low` DECIMAL(16, 8) UNSIGNED NOT NULL,\n `close` DECIMAL(16, 8) UNSIGNED NOT NULL DEFAULT 0.0,\n `volume` DECIMAL(16, 8) UNSIGNED NOT NULL DEFAULT 0.0,\n `closed` BOOL NOT NULL DEFAULT TRUE,\n `last_trade_id` INT UNSIGNED NOT NULL DEFAULT 0,\n `num_trades` INT UNSIGNED NOT NULL DEFAULT 0,\n PRIMARY KEY (`gid`)\n);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX `klines_end_time_symbol_interval` ON klines (`end_time`, `symbol`, `interval`);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `okex_klines` LIKE `klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `binance_klines` LIKE `klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `max_klines` LIKE `klines`;") + if err != nil { + return err + } + + return err +} + +func downKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX `klines_end_time_symbol_interval` ON `klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE `binance_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE `okex_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE `max_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE `klines`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20201211175751_fix_symbol_length.go b/pkg/migrations/mysql/20201211175751_fix_symbol_length.go new file mode 100644 index 000000000..4bc7b15db --- /dev/null +++ b/pkg/migrations/mysql/20201211175751_fix_symbol_length.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upFixSymbolLength, downFixSymbolLength) +} + +func upFixSymbolLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE trades MODIFY COLUMN symbol VARCHAR(9);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE orders MODIFY COLUMN symbol VARCHAR(9);") + if err != nil { + return err + } + + return err +} + +func downFixSymbolLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE trades MODIFY COLUMN symbol VARCHAR(8);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE orders MODIFY COLUMN symbol VARCHAR(8);") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20210118163847_fix_unique_index.go b/pkg/migrations/mysql/20210118163847_fix_unique_index.go new file mode 100644 index 000000000..141fdf44e --- /dev/null +++ b/pkg/migrations/mysql/20210118163847_fix_unique_index.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upFixUniqueIndex, downFixUniqueIndex) +} + +func upFixUniqueIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` DROP INDEX `id`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` ADD UNIQUE INDEX `id` (`exchange`,`symbol`, `side`, `id`);") + if err != nil { + return err + } + + return err +} + +func downFixUniqueIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` DROP INDEX `id`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` ADD UNIQUE INDEX `id` (`id`);") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20210119232826_add_margin_columns.go b/pkg/migrations/mysql/20210119232826_add_margin_columns.go new file mode 100644 index 000000000..48a096f00 --- /dev/null +++ b/pkg/migrations/mysql/20210119232826_add_margin_columns.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upAddMarginColumns, downAddMarginColumns) +} + +func upAddMarginColumns(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades`\n ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE,\n ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE\n ;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders`\n ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE,\n ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE\n ;") + if err != nil { + return err + } + + return err +} + +func downAddMarginColumns(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades`\n DROP COLUMN `is_margin`,\n DROP COLUMN `is_isolated`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders`\n DROP COLUMN `is_margin`,\n DROP COLUMN `is_isolated`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/mysql/20210129182704_trade_price_quantity_index.go b/pkg/migrations/mysql/20210129182704_trade_price_quantity_index.go new file mode 100644 index 000000000..22e5570aa --- /dev/null +++ b/pkg/migrations/mysql/20210129182704_trade_price_quantity_index.go @@ -0,0 +1,33 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradePriceQuantityIndex, downTradePriceQuantityIndex) +} + +func upTradePriceQuantityIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_price_quantity ON trades (order_id,price,quantity);") + if err != nil { + return err + } + + return err +} + +func downTradePriceQuantityIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_price_quantity ON trades") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20200721225616_trades.go b/pkg/migrations/sqlite3/20200721225616_trades.go new file mode 100644 index 000000000..d061db626 --- /dev/null +++ b/pkg/migrations/sqlite3/20200721225616_trades.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTrades, downTrades) +} + +func upTrades(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `trades`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `id` INTEGER,\n `exchange` TEXT NOT NULL DEFAULT '',\n `symbol` TEXT NOT NULL,\n `price` DECIMAL(16, 8) NOT NULL,\n `quantity` DECIMAL(16, 8) NOT NULL,\n `quote_quantity` DECIMAL(16, 8) NOT NULL,\n `fee` DECIMAL(16, 8) NOT NULL,\n `fee_currency` VARCHAR(4) NOT NULL,\n `is_buyer` BOOLEAN NOT NULL DEFAULT FALSE,\n `is_maker` BOOLEAN NOT NULL DEFAULT FALSE,\n `side` VARCHAR(4) NOT NULL DEFAULT '',\n `traded_at` DATETIME(3) NOT NULL\n);") + if err != nil { + return err + } + + return err +} + +func downTrades(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `trades`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20200819054742_trade_index.go b/pkg/migrations/sqlite3/20200819054742_trade_index.go new file mode 100644 index 000000000..c22577348 --- /dev/null +++ b/pkg/migrations/sqlite3/20200819054742_trade_index.go @@ -0,0 +1,53 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradeIndex, downTradeIndex) +} + +func upTradeIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades(symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades(symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades(traded_at, symbol);") + if err != nil { + return err + } + + return err +} + +func downTradeIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_symbol_fee_currency;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_traded_at_symbol;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201102222546_orders.go b/pkg/migrations/sqlite3/20201102222546_orders.go new file mode 100644 index 000000000..b360c575d --- /dev/null +++ b/pkg/migrations/sqlite3/20201102222546_orders.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upOrders, downOrders) +} + +func upOrders(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `orders`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `exchange` VARCHAR NOT NULL DEFAULT '',\n -- order_id is the order id returned from the exchange\n `order_id` INTEGER NOT NULL,\n `client_order_id` VARCHAR NOT NULL DEFAULT '',\n `order_type` VARCHAR NOT NULL,\n `symbol` VARCHAR NOT NULL,\n `status` VARCHAR NOT NULL,\n `time_in_force` VARCHAR NOT NULL,\n `price` DECIMAL(16, 8) NOT NULL,\n `stop_price` DECIMAL(16, 8) NOT NULL,\n `quantity` DECIMAL(16, 8) NOT NULL,\n `executed_quantity` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `side` VARCHAR NOT NULL DEFAULT '',\n `is_working` BOOLEAN NOT NULL DEFAULT FALSE,\n `created_at` DATETIME(3) NOT NULL,\n `updated_at` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP\n);") + if err != nil { + return err + } + + return err +} + +func downOrders(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `orders`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201103173342_trades_add_order_id.go b/pkg/migrations/sqlite3/20201103173342_trades_add_order_id.go new file mode 100644 index 000000000..7130b78b6 --- /dev/null +++ b/pkg/migrations/sqlite3/20201103173342_trades_add_order_id.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradesAddOrderId, downTradesAddOrderId) +} + +func upTradesAddOrderId(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` ADD COLUMN `order_id` INTEGER;") + if err != nil { + return err + } + + return err +} + +func downTradesAddOrderId(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` RENAME COLUMN `order_id` TO `order_id_deleted`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201105092857_trades_index_fix.go b/pkg/migrations/sqlite3/20201105092857_trades_index_fix.go new file mode 100644 index 000000000..05112484d --- /dev/null +++ b/pkg/migrations/sqlite3/20201105092857_trades_index_fix.go @@ -0,0 +1,83 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradesIndexFix, downTradesIndexFix) +} + +func upTradesIndexFix(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_symbol_fee_currency;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_traded_at_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades (exchange, symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades (exchange, symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades (exchange, traded_at, symbol);") + if err != nil { + return err + } + + return err +} + +func downTradesIndexFix(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_symbol_fee_currency;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS trades_traded_at_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol ON trades (symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_symbol_fee_currency ON trades (symbol, fee_currency, traded_at);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_traded_at_symbol ON trades (traded_at, symbol);") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201105093056_orders_add_index.go b/pkg/migrations/sqlite3/20201105093056_orders_add_index.go new file mode 100644 index 000000000..d880d2d0a --- /dev/null +++ b/pkg/migrations/sqlite3/20201105093056_orders_add_index.go @@ -0,0 +1,43 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upOrdersAddIndex, downOrdersAddIndex) +} + +func upOrdersAddIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX orders_symbol ON orders (exchange, symbol);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE UNIQUE INDEX orders_order_id ON orders (order_id, exchange);") + if err != nil { + return err + } + + return err +} + +func downOrdersAddIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS orders_symbol;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS orders_order_id;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201106114742_klines.go b/pkg/migrations/sqlite3/20201106114742_klines.go new file mode 100644 index 000000000..792cc7183 --- /dev/null +++ b/pkg/migrations/sqlite3/20201106114742_klines.go @@ -0,0 +1,73 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upKlines, downKlines) +} + +func upKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `klines`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `exchange` VARCHAR(10) NOT NULL,\n `start_time` DATETIME(3) NOT NULL,\n `end_time` DATETIME(3) NOT NULL,\n `interval` VARCHAR(3) NOT NULL,\n `symbol` VARCHAR(7) NOT NULL,\n `open` DECIMAL(16, 8) NOT NULL,\n `high` DECIMAL(16, 8) NOT NULL,\n `low` DECIMAL(16, 8) NOT NULL,\n `close` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `volume` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `closed` BOOLEAN NOT NULL DEFAULT TRUE,\n `last_trade_id` INT NOT NULL DEFAULT 0,\n `num_trades` INT NOT NULL DEFAULT 0\n);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE INDEX `klines_end_time_symbol_interval` ON klines (`end_time`, `symbol`, `interval`);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `okex_klines` AS SELECT * FROM `klines` WHERE 0") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `binance_klines` AS SELECT * FROM `klines` WHERE 0") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `max_klines` AS SELECT * FROM `klines` WHERE 0") + if err != nil { + return err + } + + return err +} + +func downKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS `klines_end_time_symbol_interval`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `binance_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `okex_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `max_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `klines`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20201211175751_fix_symbol_length.go b/pkg/migrations/sqlite3/20201211175751_fix_symbol_length.go new file mode 100644 index 000000000..562967063 --- /dev/null +++ b/pkg/migrations/sqlite3/20201211175751_fix_symbol_length.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upFixSymbolLength, downFixSymbolLength) +} + +func upFixSymbolLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "SELECT 1;") + if err != nil { + return err + } + + return err +} + +func downFixSymbolLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "SELECT 1;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20210118163847_fix_unique_index.go b/pkg/migrations/sqlite3/20210118163847_fix_unique_index.go new file mode 100644 index 000000000..87b1a8a91 --- /dev/null +++ b/pkg/migrations/sqlite3/20210118163847_fix_unique_index.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upFixUniqueIndex, downFixUniqueIndex) +} + +func upFixUniqueIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE UNIQUE INDEX `trade_unique_id` ON `trades` (`exchange`,`symbol`, `side`, `id`);") + if err != nil { + return err + } + + return err +} + +func downFixUniqueIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS `trade_unique_id`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20210119232826_add_margin_columns.go b/pkg/migrations/sqlite3/20210119232826_add_margin_columns.go new file mode 100644 index 000000000..f403e5ad6 --- /dev/null +++ b/pkg/migrations/sqlite3/20210119232826_add_margin_columns.go @@ -0,0 +1,63 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upAddMarginColumns, downAddMarginColumns) +} + +func upAddMarginColumns(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders` ADD COLUMN `is_margin` BOOLEAN NOT NULL DEFAULT FALSE;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders` ADD COLUMN `is_isolated` BOOLEAN NOT NULL DEFAULT FALSE;") + if err != nil { + return err + } + + return err +} + +func downAddMarginColumns(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` RENAME COLUMN `is_margin` TO `is_margin_deleted`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `trades` RENAME COLUMN `is_isolated` TO `is_isolated_deleted`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders` RENAME COLUMN `is_margin` TO `is_margin_deleted`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders` RENAME COLUMN `is_isolated` TO `is_isolated_deleted`;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20210129182704_trade_price_quantity_index.go b/pkg/migrations/sqlite3/20210129182704_trade_price_quantity_index.go new file mode 100644 index 000000000..73c608a61 --- /dev/null +++ b/pkg/migrations/sqlite3/20210129182704_trade_price_quantity_index.go @@ -0,0 +1,33 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + rockhopper.AddMigration(upTradePriceQuantityIndex, downTradePriceQuantityIndex) +} + +func upTradePriceQuantityIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE INDEX trades_price_quantity ON trades (order_id,price,quantity);") + if err != nil { + return err + } + + return err +} + +func downTradePriceQuantityIndex(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP INDEX trades_price_quantity;") + if err != nil { + return err + } + + return err +} From 63a8c00a4c3cb1fdfc451fc528e0af860873d95c Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:51:11 +0800 Subject: [PATCH 16/23] fix makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9a6ff5d5f..8dbbd46e4 100644 --- a/Makefile +++ b/Makefile @@ -28,9 +28,9 @@ dist: bin-dir bbgo-linux bbgo-darwin tar -C $(BUILD_DIR) -cvzf $(DIST_DIR)/bbgo-$$(git describe --tags).tar.gz . migrations: - rockhopper compile --config rockhopper.yaml --output pkg/migrations - git add -v pkg/migrations - git commit -m "Update migration package" pkg/migrations + rockhopper compile --config rockhopper_mysql.yaml --output pkg/migrations/mysql + rockhopper compile --config rockhopper_sqlite.yaml --output pkg/migrations/sqlite3 + git add -v pkg/migrations && git commit -m "compile and update migration package" pkg/migrations || true docker: GOPATH=$(PWD)/_mod go mod download From ecf94cdeea9f015e6afc485505129e6f86e7ddfe Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:51:35 +0800 Subject: [PATCH 17/23] .travis.yml: test make migrations --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3d513d589..427b4b326 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ install: before_script: - go mod download +- make migrations script: - bash scripts/test-sqlite3-migrations.sh From 276b6c1e48da226a1c863d78c0f1d122791a1354 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:52:53 +0800 Subject: [PATCH 18/23] drop the legacy upgradeDB --- pkg/bbgo/db.go | 29 ----------------------------- rockhopper_mysql.yaml | 2 +- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/pkg/bbgo/db.go b/pkg/bbgo/db.go index 0ff4f0d13..50431f02c 100644 --- a/pkg/bbgo/db.go +++ b/pkg/bbgo/db.go @@ -1,13 +1,9 @@ package bbgo import ( - "context" - "database/sql" - // register the go migrations _ "github.com/c9s/bbgo/pkg/migrations" - "github.com/c9s/rockhopper" "github.com/go-sql-driver/mysql" "github.com/jmoiron/sqlx" ) @@ -23,28 +19,3 @@ func ConnectMySQL(dsn string) (*sqlx.DB, error) { return sqlx.Connect("mysql", dsn) } -func upgradeDB(ctx context.Context, driver string, db *sql.DB) error { - dialect, err := rockhopper.LoadDialect(driver) - if err != nil { - return err - } - - loader := &rockhopper.GoMigrationLoader{} - migrations, err := loader.Load() - if err != nil { - return err - } - - rh := rockhopper.New(driver, dialect, db) - - currentVersion, err := rh.CurrentVersion() - if err != nil { - return err - } - - if err := rockhopper.Up(ctx, rh, migrations, currentVersion, 0); err != nil { - return err - } - - return nil -} diff --git a/rockhopper_mysql.yaml b/rockhopper_mysql.yaml index 01e86e7bf..6dde3ff23 100644 --- a/rockhopper_mysql.yaml +++ b/rockhopper_mysql.yaml @@ -2,4 +2,4 @@ driver: mysql dialect: mysql dsn: "root@tcp(localhost:3306)/bbgo?parseTime=true" -migrationsDir: migrations +migrationsDir: migrations/mysql From 99b56003ebebf68716b014572f04c6a082ae2e25 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 09:56:26 +0800 Subject: [PATCH 19/23] clean up legacy db connection handling with the new database service --- pkg/bbgo/db.go | 15 --------------- pkg/cmd/backtest.go | 14 +++++--------- pkg/cmd/pnl.go | 18 ++++++++++-------- pkg/server/setup.go | 4 ++-- 4 files changed, 17 insertions(+), 34 deletions(-) diff --git a/pkg/bbgo/db.go b/pkg/bbgo/db.go index 50431f02c..dcdb0eaec 100644 --- a/pkg/bbgo/db.go +++ b/pkg/bbgo/db.go @@ -3,19 +3,4 @@ package bbgo import ( // register the go migrations _ "github.com/c9s/bbgo/pkg/migrations" - - "github.com/go-sql-driver/mysql" - "github.com/jmoiron/sqlx" ) - -func ConnectMySQL(dsn string) (*sqlx.DB, error) { - config, err := mysql.ParseDSN(dsn) - if err != nil { - return nil, err - } - - config.ParseTime = true - dsn = config.FormatDSN() - return sqlx.Connect("mysql", dsn) -} - diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 638a4885d..8816302d3 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -3,12 +3,12 @@ package cmd import ( "context" "fmt" + "os" "time" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/c9s/bbgo/pkg/accounting/pnl" "github.com/c9s/bbgo/pkg/backtest" @@ -96,10 +96,6 @@ var BacktestCmd = &cobra.Command{ return err } - db, err := bbgo.ConnectMySQL(viper.GetString("mysql-url")) - if err != nil { - return err - } if userConfig.Backtest == nil { return errors.New("backtest config is not defined") @@ -116,14 +112,14 @@ var BacktestCmd = &cobra.Command{ } environ := bbgo.NewEnvironment() - if viper.IsSet("mysql-url") { - dsn := viper.GetString("mysql-url") - if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { + if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { + err := environ.ConfigureDatabase(ctx, "mysql", dsn) + if err != nil { return err } } - backtestService := &service.BacktestService{DB: db} + backtestService := &service.BacktestService{DB: environ.DatabaseService.DB} if wantSync { log.Info("starting synchronization...") diff --git a/pkg/cmd/pnl.go b/pkg/cmd/pnl.go index 9e27c6f5b..82048a10e 100644 --- a/pkg/cmd/pnl.go +++ b/pkg/cmd/pnl.go @@ -2,13 +2,13 @@ package cmd import ( "context" + "os" "strings" "time" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/c9s/bbgo/pkg/accounting" "github.com/c9s/bbgo/pkg/accounting/pnl" @@ -51,20 +51,22 @@ var PnLCmd = &cobra.Command{ return err } - db, err := bbgo.ConnectMySQL(viper.GetString("mysql-url")) - if err != nil { - return err - } - tradeService := &service.TradeService{DB: db} + environ := bbgo.NewEnvironment() + if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { + err := environ.ConfigureDatabase(ctx, "mysql", dsn) + if err != nil { + return err + } + } var trades []types.Trade tradingFeeCurrency := exchange.PlatformFeeCurrency() if strings.HasPrefix(symbol, tradingFeeCurrency) { log.Infof("loading all trading fee currency related trades: %s", symbol) - trades, err = tradeService.QueryForTradingFeeCurrency(exchange.Name(), symbol, tradingFeeCurrency) + trades, err = environ.TradeService.QueryForTradingFeeCurrency(exchange.Name(), symbol, tradingFeeCurrency) } else { - trades, err = tradeService.Query(service.QueryTradesOptions{ + trades, err = environ.TradeService.Query(service.QueryTradesOptions{ Exchange: exchange.Name(), Symbol: symbol, }) diff --git a/pkg/server/setup.go b/pkg/server/setup.go index 75d3a5788..6619429ad 100644 --- a/pkg/server/setup.go +++ b/pkg/server/setup.go @@ -2,6 +2,7 @@ package server import ( "context" + "database/sql" "net/http" "os" "syscall" @@ -29,7 +30,7 @@ func (s *Server) setupTestDB(c *gin.Context) { return } - db, err := bbgo.ConnectMySQL(dsn) + db, err := sql.Open("mysql", dsn) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return @@ -142,4 +143,3 @@ func (s *Server) setupRestart(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"success": true}) } - From 51d399a49d6e2e82f8768c3913124121db8f751d Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 11:17:31 +0800 Subject: [PATCH 20/23] upgrade rockhopper --- go.mod | 4 ++-- go.sum | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 4a4033a8a..8fb3096d3 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.13 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/adshao/go-binance/v2 v2.2.1-0.20210119141603-20ceb26d876b - github.com/c9s/rockhopper v1.2.1-0.20210115022144-cc77e66fc34f + github.com/c9s/rockhopper v1.2.1-0.20210206025705-bbb1e34bd7a9 github.com/codingconcepts/env v0.0.0-20200821220118-a8fbf8d84482 github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect github.com/gin-contrib/cors v1.3.1 @@ -53,7 +53,7 @@ require ( github.com/x-cray/logrus-prefixed-formatter v0.5.2 github.com/zserge/lorca v0.1.9 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect - golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect golang.org/x/text v0.3.5 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 gonum.org/v1/gonum v0.8.1 diff --git a/go.sum b/go.sum index 982d91dba..7db55f485 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8 github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/c9s/rockhopper v1.2.1-0.20210115022144-cc77e66fc34f h1:n1Ly7178MJj+GQB38q4dV66QktUvzEi2rA7xCtTy6Ck= github.com/c9s/rockhopper v1.2.1-0.20210115022144-cc77e66fc34f/go.mod h1:KJnQjZSrWA83jjwGF/+O7Y96VCVirYTYEvXJJOc6kMU= +github.com/c9s/rockhopper v1.2.1-0.20210206025705-bbb1e34bd7a9 h1:umJ5T1aKfA6zmHTJffe5axqM9mtr/tscllnX2wnZzBA= +github.com/c9s/rockhopper v1.2.1-0.20210206025705-bbb1e34bd7a9/go.mod h1:KJnQjZSrWA83jjwGF/+O7Y96VCVirYTYEvXJJOc6kMU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -240,6 +242,7 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -466,6 +469,8 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From dd9dbee90303f5b873eb6143b761e06bf4d6ae3e Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 11:33:49 +0800 Subject: [PATCH 21/23] refactor database configuration with env vars --- pkg/cmd/backtest.go | 29 ++++++++++++++++++++++++----- pkg/cmd/cancel.go | 9 ++------- pkg/cmd/pnl.go | 8 ++------ pkg/cmd/run.go | 6 ++---- pkg/cmd/sync.go | 9 ++------- pkg/server/setup.go | 3 ++- pkg/service/database.go | 3 ++- 7 files changed, 36 insertions(+), 31 deletions(-) diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 8816302d3..425a4be36 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -112,11 +112,9 @@ var BacktestCmd = &cobra.Command{ } environ := bbgo.NewEnvironment() - if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - err := environ.ConfigureDatabase(ctx, "mysql", dsn) - if err != nil { - return err - } + + if err := configureDB(ctx, environ) ; err != nil { + return err } backtestService := &service.BacktestService{DB: environ.DatabaseService.DB} @@ -289,3 +287,24 @@ func InBaseAsset(balances types.BalanceMap, market types.Market, price float64) base := balances[market.BaseCurrency] return (base.Locked.Float64() + base.Available.Float64()) + ((quote.Locked.Float64() + quote.Available.Float64()) / price) } + +// configureDB configures the database service based on the environment variable +func configureDB(ctx context.Context, environ *bbgo.Environment) error { + if driver, ok := os.LookupEnv("DB_DRIVER") ; ok { + + if dsn, ok := os.LookupEnv("DB_DSN") ; ok { + return environ.ConfigureDatabase(ctx, driver, dsn) + } + + } else if dsn, ok := os.LookupEnv("SQLITE3_DSN"); ok { + + return environ.ConfigureDatabase(ctx, "sqlite3", dsn) + + } else if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { + + return environ.ConfigureDatabase(ctx, "mysql", dsn) + + } + + return nil +} diff --git a/pkg/cmd/cancel.go b/pkg/cmd/cancel.go index f64971d8e..af4526587 100644 --- a/pkg/cmd/cancel.go +++ b/pkg/cmd/cancel.go @@ -3,7 +3,6 @@ package cmd import ( "context" "fmt" - "os" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -63,12 +62,8 @@ var CancelCmd = &cobra.Command{ } environ := bbgo.NewEnvironment() - - if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - err := environ.ConfigureDatabase(ctx, "mysql", dsn) - if err != nil { - return err - } + if err := configureDB(ctx, environ) ; err != nil { + return err } if err := environ.AddExchangesFromConfig(userConfig); err != nil { diff --git a/pkg/cmd/pnl.go b/pkg/cmd/pnl.go index 82048a10e..215a54ba6 100644 --- a/pkg/cmd/pnl.go +++ b/pkg/cmd/pnl.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "os" "strings" "time" @@ -53,11 +52,8 @@ var PnLCmd = &cobra.Command{ environ := bbgo.NewEnvironment() - if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - err := environ.ConfigureDatabase(ctx, "mysql", dsn) - if err != nil { - return err - } + if err := configureDB(ctx, environ) ; err != nil { + return err } var trades []types.Trade diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 90eea1123..7c042ff4a 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -96,10 +96,8 @@ func runSetup(baseCtx context.Context, userConfig *bbgo.Config, enableApiServer } func BootstrapEnvironment(ctx context.Context, environ *bbgo.Environment, userConfig *bbgo.Config) error { - if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { - return err - } + if err := configureDB(ctx, environ) ; err != nil { + return err } if err := environ.AddExchangesFromConfig(userConfig); err != nil { diff --git a/pkg/cmd/sync.go b/pkg/cmd/sync.go index 8f0b0314f..05bc867fd 100644 --- a/pkg/cmd/sync.go +++ b/pkg/cmd/sync.go @@ -9,7 +9,6 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/c9s/bbgo/pkg/bbgo" ) @@ -52,12 +51,8 @@ var SyncCmd = &cobra.Command{ } environ := bbgo.NewEnvironment() - - if viper.IsSet("mysql-url") { - dsn := viper.GetString("mysql-url") - if err := environ.ConfigureDatabase(ctx, "mysql", dsn); err != nil { - return err - } + if err := configureDB(ctx, environ) ; err != nil { + return err } if err := environ.AddExchangesFromConfig(userConfig); err != nil { diff --git a/pkg/server/setup.go b/pkg/server/setup.go index 6619429ad..d81a7f9d4 100644 --- a/pkg/server/setup.go +++ b/pkg/server/setup.go @@ -59,7 +59,8 @@ func (s *Server) setupConfigureDB(c *gin.Context) { return } - if err := s.Environ.ConfigureDatabase(c, "mysql", dsn); err != nil { + driver := "mysql" + if err := s.Environ.ConfigureDatabase(c, driver, dsn); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } diff --git a/pkg/service/database.go b/pkg/service/database.go index 47d797b75..05f5b5bcb 100644 --- a/pkg/service/database.go +++ b/pkg/service/database.go @@ -48,7 +48,7 @@ func (s *DatabaseService) Upgrade(ctx context.Context) error { } loader := &rockhopper.GoMigrationLoader{} - migrations, err := loader.Load() + migrations, err := loader.LoadByPackageSuffix(s.Driver) if err != nil { return err } @@ -74,6 +74,7 @@ func ReformatMysqlDSN(dsn string) (string, error) { return "", err } + // we need timestamp and datetime fields to be parsed into time.Time struct config.ParseTime = true dsn = config.FormatDSN() return dsn, nil From 0b657d59f99243cb06e41331dd8a16cbd8a4654d Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 11:34:53 +0800 Subject: [PATCH 22/23] make inBaseAsset as private method --- pkg/cmd/backtest.go | 31 ++----------------------------- pkg/cmd/utils.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 29 deletions(-) create mode 100644 pkg/cmd/utils.go diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 425a4be36..f5765acf9 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -3,7 +3,6 @@ package cmd import ( "context" "fmt" - "os" "time" "github.com/pkg/errors" @@ -267,8 +266,8 @@ var BacktestCmd = &cobra.Command{ finalBalances.Print() if wantBaseAssetBaseline { - initBaseAsset := InBaseAsset(initBalances, market, startPrice) - finalBaseAsset := InBaseAsset(finalBalances, market, lastPrice) + initBaseAsset := inBaseAsset(initBalances, market, startPrice) + finalBaseAsset := inBaseAsset(finalBalances, market, lastPrice) log.Infof("INITIAL ASSET ~= %s %s (1 %s = %f)", market.FormatQuantity(initBaseAsset), market.BaseCurrency, market.BaseCurrency, startPrice) log.Infof("FINAL ASSET ~= %s %s (1 %s = %f)", market.FormatQuantity(finalBaseAsset), market.BaseCurrency, market.BaseCurrency, lastPrice) @@ -282,29 +281,3 @@ var BacktestCmd = &cobra.Command{ }, } -func InBaseAsset(balances types.BalanceMap, market types.Market, price float64) float64 { - quote := balances[market.QuoteCurrency] - base := balances[market.BaseCurrency] - return (base.Locked.Float64() + base.Available.Float64()) + ((quote.Locked.Float64() + quote.Available.Float64()) / price) -} - -// configureDB configures the database service based on the environment variable -func configureDB(ctx context.Context, environ *bbgo.Environment) error { - if driver, ok := os.LookupEnv("DB_DRIVER") ; ok { - - if dsn, ok := os.LookupEnv("DB_DSN") ; ok { - return environ.ConfigureDatabase(ctx, driver, dsn) - } - - } else if dsn, ok := os.LookupEnv("SQLITE3_DSN"); ok { - - return environ.ConfigureDatabase(ctx, "sqlite3", dsn) - - } else if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { - - return environ.ConfigureDatabase(ctx, "mysql", dsn) - - } - - return nil -} diff --git a/pkg/cmd/utils.go b/pkg/cmd/utils.go new file mode 100644 index 000000000..9298cc659 --- /dev/null +++ b/pkg/cmd/utils.go @@ -0,0 +1,36 @@ +package cmd + +import ( + "context" + "os" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/types" +) + +func inBaseAsset(balances types.BalanceMap, market types.Market, price float64) float64 { + quote := balances[market.QuoteCurrency] + base := balances[market.BaseCurrency] + return (base.Locked.Float64() + base.Available.Float64()) + ((quote.Locked.Float64() + quote.Available.Float64()) / price) +} + +// configureDB configures the database service based on the environment variable +func configureDB(ctx context.Context, environ *bbgo.Environment) error { + if driver, ok := os.LookupEnv("DB_DRIVER"); ok { + + if dsn, ok := os.LookupEnv("DB_DSN"); ok { + return environ.ConfigureDatabase(ctx, driver, dsn) + } + + } else if dsn, ok := os.LookupEnv("SQLITE3_DSN"); ok { + + return environ.ConfigureDatabase(ctx, "sqlite3", dsn) + + } else if dsn, ok := os.LookupEnv("MYSQL_URL"); ok { + + return environ.ConfigureDatabase(ctx, "mysql", dsn) + + } + + return nil +} From 32117af4b047779ddf6643ddc3648875bbbb97bd Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 6 Feb 2021 11:44:49 +0800 Subject: [PATCH 23/23] service: remove the ignore keyword to make the sql compatible with sqlite3 --- pkg/service/trade.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/service/trade.go b/pkg/service/trade.go index 6739fbf07..e607cc668 100644 --- a/pkg/service/trade.go +++ b/pkg/service/trade.go @@ -244,7 +244,7 @@ func (s *TradeService) scanRows(rows *sqlx.Rows) (trades []types.Trade, err erro func (s *TradeService) Insert(trade types.Trade) error { _, err := s.DB.NamedExec(` - INSERT IGNORE INTO trades (id, exchange, order_id, symbol, price, quantity, quote_quantity, side, is_buyer, is_maker, fee, fee_currency, traded_at, is_margin, is_isolated) + INSERT INTO trades (id, exchange, order_id, symbol, price, quantity, quote_quantity, side, is_buyer, is_maker, fee, fee_currency, traded_at, is_margin, is_isolated) VALUES (:id, :exchange, :order_id, :symbol, :price, :quantity, :quote_quantity, :side, :is_buyer, :is_maker, :fee, :fee_currency, :traded_at, :is_margin, :is_isolated)`, trade) return err