From 73182bb2ddc41e05067bc317265ed445d5a2196a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Aug 2020 14:50:57 +0200 Subject: [PATCH] Update migrations to populate Orders table for open orders --- freqtrade/persistence/migrations.py | 201 ++++++++++++++++------------ freqtrade/persistence/models.py | 10 +- tests/test_persistence.py | 34 +++-- 3 files changed, 148 insertions(+), 97 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 1bce8fef2..55825436d 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -5,6 +5,7 @@ from sqlalchemy import inspect logger = logging.getLogger(__name__) + def get_table_names_for_table(inspector, tabletype): return [t for t in inspector.get_table_names() if t.startswith(tabletype)] @@ -17,7 +18,110 @@ def get_column_def(columns: List, column: str, default: str) -> str: return default if not has_column(columns, column) else column -def check_migrate(engine, decl_base) -> None: +def get_backup_name(tabs, backup_prefix: str): + table_back_name = backup_prefix + for i, table_back_name in enumerate(tabs): + table_back_name = f'{backup_prefix}{i}' + logger.debug(f'trying {table_back_name}') + + return table_back_name + + +def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List): + fee_open = get_column_def(cols, 'fee_open', 'fee') + fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') + fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') + fee_close = get_column_def(cols, 'fee_close', 'fee') + fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') + fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') + open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') + close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') + stop_loss = get_column_def(cols, 'stop_loss', '0.0') + stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') + initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') + initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') + stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') + stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') + max_rate = get_column_def(cols, 'max_rate', '0.0') + min_rate = get_column_def(cols, 'min_rate', 'null') + sell_reason = get_column_def(cols, 'sell_reason', 'null') + strategy = get_column_def(cols, 'strategy', 'null') + # If ticker-interval existed use that, else null. + if has_column(cols, 'ticker_interval'): + timeframe = get_column_def(cols, 'timeframe', 'ticker_interval') + else: + timeframe = get_column_def(cols, 'timeframe', 'null') + + open_trade_price = get_column_def(cols, 'open_trade_price', + f'amount * open_rate * (1 + {fee_open})') + close_profit_abs = get_column_def( + cols, 'close_profit_abs', + f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") + sell_order_status = get_column_def(cols, 'sell_order_status', 'null') + amount_requested = get_column_def(cols, 'amount_requested', 'amount') + + # Schema migration necessary + engine.execute(f"alter table trades rename to {table_back_name}") + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + engine.execute(f"drop index {index['name']}") + # let SQLAlchemy create the schema as required + decl_base.metadata.create_all(engine) + + # Copy data back - following the correct schema + engine.execute(f"""insert into trades + (id, exchange, pair, is_open, + fee_open, fee_open_cost, fee_open_currency, + fee_close, fee_close_cost, fee_open_currency, open_rate, + open_rate_requested, close_rate, close_rate_requested, close_profit, + stake_amount, amount, amount_requested, open_date, close_date, open_order_id, + stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, + stoploss_order_id, stoploss_last_update, + max_rate, min_rate, sell_reason, sell_order_status, strategy, + timeframe, open_trade_price, close_profit_abs + ) + select id, lower(exchange), + case + when instr(pair, '_') != 0 then + substr(pair, instr(pair, '_') + 1) || '/' || + substr(pair, 1, instr(pair, '_') - 1) + else pair + end + pair, + is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, + {fee_open_currency} fee_open_currency, {fee_close} fee_close, + {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, + open_rate, {open_rate_requested} open_rate_requested, close_rate, + {close_rate_requested} close_rate_requested, close_profit, + stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, + {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, + {initial_stop_loss} initial_stop_loss, + {initial_stop_loss_pct} initial_stop_loss_pct, + {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, + {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, + {sell_order_status} sell_order_status, + {strategy} strategy, {timeframe} timeframe, + {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs + from {table_back_name} + """) + + +def migrate_open_orders_to_trades(engine): + engine.execute(""" + insert into orders (trade_id, order_id, ft_order_side) + select id, open_order_id, + case when close_rate_requested is null then 'buy' + else 'sell' end ft_order_side + from trades + where open_order_id is not null + union all + select id, stoploss_order_id, 'stoploss' + from trades + where stoploss_order_id is not null + """) + + +def check_migrate(engine, decl_base, previous_tables) -> None: """ Checks if migration is necessary and migrates if necessary """ @@ -25,92 +129,21 @@ def check_migrate(engine, decl_base) -> None: cols = inspector.get_columns('trades') tabs = get_table_names_for_table(inspector, 'trades') - table_back_name = 'trades_bak' - for i, table_back_name in enumerate(tabs): - table_back_name = f'trades_bak{i}' - logger.debug(f'trying {table_back_name}') + table_back_name = get_backup_name(tabs, 'trades_bak') # Check for latest column if not has_column(cols, 'amount_requested'): - logger.info(f'Running database migration - backup available as {table_back_name}') - - fee_open = get_column_def(cols, 'fee_open', 'fee') - fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') - fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') - fee_close = get_column_def(cols, 'fee_close', 'fee') - fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') - fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') - open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') - close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') - stop_loss = get_column_def(cols, 'stop_loss', '0.0') - stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') - initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') - initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') - stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') - stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') - max_rate = get_column_def(cols, 'max_rate', '0.0') - min_rate = get_column_def(cols, 'min_rate', 'null') - sell_reason = get_column_def(cols, 'sell_reason', 'null') - strategy = get_column_def(cols, 'strategy', 'null') - # If ticker-interval existed use that, else null. - if has_column(cols, 'ticker_interval'): - timeframe = get_column_def(cols, 'timeframe', 'ticker_interval') - else: - timeframe = get_column_def(cols, 'timeframe', 'null') - - open_trade_price = get_column_def(cols, 'open_trade_price', - f'amount * open_rate * (1 + {fee_open})') - close_profit_abs = get_column_def( - cols, 'close_profit_abs', - f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") - sell_order_status = get_column_def(cols, 'sell_order_status', 'null') - amount_requested = get_column_def(cols, 'amount_requested', 'amount') - - # Schema migration necessary - engine.execute(f"alter table trades rename to {table_back_name}") - # drop indexes on backup table - for index in inspector.get_indexes(table_back_name): - engine.execute(f"drop index {index['name']}") - # let SQLAlchemy create the schema as required - decl_base.metadata.create_all(engine) - - # Copy data back - following the correct schema - engine.execute(f"""insert into trades - (id, exchange, pair, is_open, - fee_open, fee_open_cost, fee_open_currency, - fee_close, fee_close_cost, fee_open_currency, open_rate, - open_rate_requested, close_rate, close_rate_requested, close_profit, - stake_amount, amount, amount_requested, open_date, close_date, open_order_id, - stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, - stoploss_order_id, stoploss_last_update, - max_rate, min_rate, sell_reason, sell_order_status, strategy, - timeframe, open_trade_price, close_profit_abs - ) - select id, lower(exchange), - case - when instr(pair, '_') != 0 then - substr(pair, instr(pair, '_') + 1) || '/' || - substr(pair, 1, instr(pair, '_') - 1) - else pair - end - pair, - is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, - {fee_open_currency} fee_open_currency, {fee_close} fee_close, - {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, - open_rate, {open_rate_requested} open_rate_requested, close_rate, - {close_rate_requested} close_rate_requested, close_profit, - stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, - {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, - {initial_stop_loss} initial_stop_loss, - {initial_stop_loss_pct} initial_stop_loss_pct, - {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, - {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, - {sell_order_status} sell_order_status, - {strategy} strategy, {timeframe} timeframe, - {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs - from {table_back_name} - """) - + logger.info(f'Running database migration for trades - backup: {table_back_name}') + migrate_trades_table(decl_base, inspector, engine, table_back_name, cols) # Reread columns - the above recreated the table! inspector = inspect(engine) cols = inspector.get_columns('trades') + + if 'orders' not in previous_tables: + logger.info('Moving open orders to Orders table.') + migrate_open_orders_to_trades(engine) + else: + logger.info(f'Running database migration for orders - backup: {table_back_name}') + pass + # Empty for now - as there is only one iteration of the orders table so far. + # table_back_name = get_backup_name(tabs, 'orders_bak') diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 12bbd8adc..b0bde02fe 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -7,8 +7,8 @@ from decimal import Decimal from typing import Any, Dict, List, Optional import arrow -from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, ForeignKey, - create_engine, desc, func) +from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, + String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Query, relationship @@ -61,9 +61,9 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: # Copy session attributes to order object too Order.session = Trade.session Order.query = Order.session.query_property() - + previous_tables = inspect(engine).get_table_names() _DECL_BASE.metadata.create_all(engine) - check_migrate(engine, decl_base=_DECL_BASE) + check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables) # Clean dry_run DB if the db is not in-memory if clean_open_orders and db_url != 'sqlite://': @@ -117,7 +117,7 @@ class Order(_DECL_BASE): filled = Column(Float, nullable=True) remaining = Column(Float, nullable=True) cost = Column(Float, nullable=True) - order_date = Column(DateTime, nullable=False, default=datetime.utcnow) + order_date = Column(DateTime, nullable=True, default=datetime.utcnow) order_filled_date = Column(DateTime, nullable=True) order_update_date = Column(DateTime, nullable=True) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index dbb62e636..c812c496f 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -8,7 +8,7 @@ from sqlalchemy import create_engine from freqtrade import constants from freqtrade.exceptions import OperationalException -from freqtrade.persistence import Trade, clean_dry_run_db, init +from freqtrade.persistence import Trade, Order, clean_dry_run_db, init from tests.conftest import log_has, create_mock_trades @@ -421,9 +421,9 @@ def test_migrate_old(mocker, default_conf, fee): PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" - insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, + insert_table_old = """INSERT INTO trades (exchange, pair, is_open, open_order_id, fee, open_rate, stake_amount, amount, open_date) - VALUES ('BITTREX', 'BTC_ETC', 1, {fee}, + VALUES ('BITTREX', 'BTC_ETC', 1, '123123', {fee}, 0.00258580, {stake}, {amount}, '2017-11-28 12:44:24.000000') """.format(fee=fee.return_value, @@ -481,6 +481,12 @@ def test_migrate_old(mocker, default_conf, fee): assert pytest.approx(trade.close_profit_abs) == trade.calc_profit() assert trade.sell_order_status is None + # Should've created one order + assert len(Order.query.all()) == 1 + order = Order.query.first() + assert order.order_id == '123123' + assert order.ft_order_side == 'buy' + def test_migrate_new(mocker, default_conf, fee, caplog): """ @@ -509,16 +515,19 @@ def test_migrate_new(mocker, default_conf, fee, caplog): sell_reason VARCHAR, strategy VARCHAR, ticker_interval INTEGER, + stoploss_order_id VARCHAR, PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date, - stop_loss, initial_stop_loss, max_rate, ticker_interval) + stop_loss, initial_stop_loss, max_rate, ticker_interval, + open_order_id, stoploss_order_id) VALUES ('binance', 'ETC/BTC', 1, {fee}, 0.00258580, {stake}, {amount}, '2019-11-28 12:44:24.000000', - 0.0, 0.0, 0.0, '5m') + 0.0, 0.0, 0.0, '5m', + 'buy_order', 'stop_order_id222') """.format(fee=fee.return_value, stake=default_conf.get("stake_amount"), amount=amount @@ -558,14 +567,23 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.sell_reason is None assert trade.strategy is None assert trade.timeframe == '5m' - assert trade.stoploss_order_id is None + assert trade.stoploss_order_id == 'stop_order_id222' assert trade.stoploss_last_update is None assert log_has("trying trades_bak1", caplog) assert log_has("trying trades_bak2", caplog) - assert log_has("Running database migration - backup available as trades_bak2", caplog) + assert log_has("Running database migration for trades - backup: trades_bak2", caplog) assert trade.open_trade_price == trade._calc_open_trade_price() assert trade.close_profit_abs is None + assert log_has("Moving open orders to Orders table.", caplog) + orders = Order.query.all() + assert len(orders) == 2 + assert orders[0].order_id == 'buy_order' + assert orders[0].ft_order_side == 'buy' + + assert orders[1].order_id == 'stop_order_id222' + assert orders[1].ft_order_side == 'stoploss' + def test_migrate_mid_state(mocker, default_conf, fee, caplog): """ @@ -626,7 +644,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): assert trade.initial_stop_loss == 0.0 assert trade.open_trade_price == trade._calc_open_trade_price() assert log_has("trying trades_bak0", caplog) - assert log_has("Running database migration - backup available as trades_bak0", caplog) + assert log_has("Running database migration for trades - backup: trades_bak0", caplog) def test_adjust_stop_loss(fee):