From bd29b7d03136b3905152b079f872d030284e7a3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Mar 2019 15:21:05 +0100 Subject: [PATCH 1/6] Test that dataprovider is loaded to strategy --- freqtrade/tests/test_freqtradebot.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index e4f0415f7..fa50b36e6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -13,12 +13,14 @@ import requests from freqtrade import (DependencyException, OperationalException, TemporaryError, constants) +from freqtrade.data.dataprovider import DataProvider from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.state import State -from freqtrade.strategy.interface import SellType, SellCheckTuple -from freqtrade.tests.conftest import log_has, log_has_re, patch_exchange, patch_edge, patch_wallet +from freqtrade.strategy.interface import SellCheckTuple, SellType +from freqtrade.tests.conftest import (log_has, log_has_re, patch_edge, + patch_exchange, patch_wallet) # Functions for recurrent object patching @@ -88,6 +90,10 @@ def test_worker_running(mocker, default_conf, caplog) -> None: assert state is State.RUNNING assert log_has('Changing state to: RUNNING', caplog.record_tuples) assert mock_throttle.call_count == 1 + # Check strategy is loaded, and received a dataprovider object + assert freqtrade.strategy + assert freqtrade.strategy.dp + assert isinstance(freqtrade.strategy.dp, DataProvider) def test_worker_stopped(mocker, default_conf, caplog) -> None: From 226fc3d99b97b9a62a2c2a8246c60d8753c367e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Mar 2019 15:23:14 +0100 Subject: [PATCH 2/6] Check that dataprovider is part of strategy --- freqtrade/tests/optimize/test_backtesting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 40754cfbc..d0b21b8f4 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -16,6 +16,7 @@ from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import evaluate_result_multi from freqtrade.data.converter import parse_ticker_dataframe +from freqtrade.data.dataprovider import DataProvider from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) @@ -346,6 +347,7 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None: assert callable(backtesting.strategy.tickerdata_to_dataframe) assert callable(backtesting.advise_buy) assert callable(backtesting.advise_sell) + assert isinstance(backtesting.strategy.dp, DataProvider) get_fee.assert_called() assert backtesting.fee == 0.5 assert not backtesting.strategy.order_types["stoploss_on_exchange"] From 0ae81d41156f84828d9bab0ec22f0686fb4d75a2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Mar 2019 15:24:47 +0100 Subject: [PATCH 3/6] Provide dataprovider access during backtesting --- freqtrade/optimize/backtesting.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 031b490c8..7c98bd3a7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -18,6 +18,7 @@ from freqtrade import DependencyException, constants from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.data import history +from freqtrade.data.dataprovider import DataProvider from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -64,6 +65,13 @@ class Backtesting(object): self.config['exchange']['uid'] = '' self.config['dry_run'] = True self.strategylist: List[IStrategy] = [] + + exchange_name = self.config.get('exchange', {}).get('name', 'bittrex').title() + self.exchange = ExchangeResolver(exchange_name, self.config).exchange + self.fee = self.exchange.get_fee() + self.dataprovider = DataProvider(self.config, self.exchange) + IStrategy.dp = self.dataprovider + if self.config.get('strategy_list', None): # Force one interval self.ticker_interval = str(self.config.get('ticker_interval')) @@ -78,15 +86,13 @@ class Backtesting(object): self.strategylist.append(StrategyResolver(self.config).strategy) # Load one strategy self._set_strategy(self.strategylist[0]) - exchange_name = self.config.get('exchange', {}).get('name', 'bittrex').title() - self.exchange = ExchangeResolver(exchange_name, self.config).exchange - self.fee = self.exchange.get_fee() def _set_strategy(self, strategy): """ Load strategy into backtesting """ self.strategy = strategy + self.ticker_interval = self.config.get('ticker_interval') self.ticker_interval_mins = constants.TICKER_INTERVAL_MINUTES[self.ticker_interval] self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe From 4cf7282027e56277f79a317a8d8a7aa3092f17be Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Mar 2019 19:31:10 +0100 Subject: [PATCH 4/6] Update dataprovider docs --- docs/bot-optimization.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index c01a0a03d..ae6377bf5 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -260,12 +260,9 @@ class Awesomestrategy(IStrategy): The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. -!!! Note - The DataProvier is currently not available during backtesting / hyperopt, but this is planned for the future. - All methods return `None` in case of failure (do not raise an exception). -Please always check if the `DataProvider` is available to avoid failures during backtesting. +Please always the mode of operation to select the correct method to get data (samples see below). #### Possible options for DataProvider @@ -292,6 +289,9 @@ if self.dp: Be carefull when using dataprovider in backtesting. `historic_ohlcv()` provides the full time-range in one go, so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). +!!! Warning Warning in hyperopt + This option should only be used in `populate_indicators()` - since it pulls the historic data from disk each time, which would be a huge performance penalty during hyperopt. + #### Available Pairs ``` python From e085fd9e9538c18a6e4e6ea51c0dd151eb31c15e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Mar 2019 19:49:58 +0100 Subject: [PATCH 5/6] Disable dataprovider from hyperopt. Dataprovider uses weak links to initialize, which cannot be pickled, and therefore cannot be used during hyperopt. --- docs/bot-optimization.md | 2 +- freqtrade/optimize/backtesting.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index ae6377bf5..f018d92a7 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -290,7 +290,7 @@ if self.dp: so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). !!! Warning Warning in hyperopt - This option should only be used in `populate_indicators()` - since it pulls the historic data from disk each time, which would be a huge performance penalty during hyperopt. + This option cannot currently be used during hyperopt. #### Available Pairs diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 7c98bd3a7..293511fc0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -69,8 +69,10 @@ class Backtesting(object): exchange_name = self.config.get('exchange', {}).get('name', 'bittrex').title() self.exchange = ExchangeResolver(exchange_name, self.config).exchange self.fee = self.exchange.get_fee() - self.dataprovider = DataProvider(self.config, self.exchange) - IStrategy.dp = self.dataprovider + + if self.config.get('runmode') != RunMode.HYPEROPT: + self.dataprovider = DataProvider(self.config, self.exchange) + IStrategy.dp = self.dataprovider if self.config.get('strategy_list', None): # Force one interval From 3bdc7b9a8880a23cf9ae0925a0194f373e027c27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Mar 2019 10:51:13 +0100 Subject: [PATCH 6/6] add missed "check" in docs --- docs/bot-optimization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index f018d92a7..9e754c213 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -262,7 +262,7 @@ The strategy provides access to the `DataProvider`. This allows you to get addit All methods return `None` in case of failure (do not raise an exception). -Please always the mode of operation to select the correct method to get data (samples see below). +Please always check the mode of operation to select the correct method to get data (samples see below). #### Possible options for DataProvider