diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index c4fc55811..d5bc76c65 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -324,62 +324,9 @@ class Awesomestrategy(IStrategy): !!! Note If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. -### Additional data (DataProvider) +*** -The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. - -All methods return `None` in case of failure (do not raise an exception). - -Please always check the mode of operation to select the correct method to get data (samples see below). - -#### Possible options for DataProvider - -- `available_pairs` - Property with tuples listing cached pairs with their intervals (pair, interval). -- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame. -- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. -- `get_pair_dataframe(pair, timeframe)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). -- `orderbook(pair, maximum)` - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries. -- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure. -- `runmode` - Property containing the current runmode. - -#### Example: fetch live / historical candle (OHLCV) data for the first informative pair - -``` python -if self.dp: - inf_pair, inf_timeframe = self.informative_pairs()[0] - informative = self.dp.get_pair_dataframe(pair=inf_pair, - timeframe=inf_timeframe) -``` - -!!! Warning "Warning about backtesting" - Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` - for the backtesting runmode) provides the full time-range in one go, - so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). - -!!! Warning "Warning in hyperopt" - This option cannot currently be used during hyperopt. - -#### Orderbook - -``` python -if self.dp: - if self.dp.runmode.value in ('live', 'dry_run'): - ob = self.dp.orderbook(metadata['pair'], 1) - dataframe['best_bid'] = ob['bids'][0][0] - dataframe['best_ask'] = ob['asks'][0][0] -``` - -!!! Warning - The order book is not part of the historic data which means backtesting and hyperopt will not work if this - method is used. - -#### Available Pairs - -``` python -if self.dp: - for pair, timeframe in self.dp.available_pairs: - print(f"available {pair}, {timeframe}") -``` +### Additional data (informative_pairs) #### Get data for non-tradeable pairs @@ -404,6 +351,107 @@ def informative_pairs(self): It is however better to use resampling to longer time-intervals when possible to avoid hammering the exchange with too many requests and risk being blocked. +*** + +### Additional data (DataProvider) + +The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. + +All methods return `None` in case of failure (do not raise an exception). + +Please always check the mode of operation to select the correct method to get data (samples see below). + +#### Possible options for DataProvider + +- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their intervals (pair, interval). +- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (ie. VolumePairlist) +- [`get_pair_dataframe(pair, timeframe)`](#get_pair_dataframepair-timeframe) - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). +- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. +- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure. +- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame. +- [`orderbook(pair, maximum)`](#orderbookpair-maximum) - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries. +- `runmode` - Property containing the current runmode. + +#### Example Usages: + +#### *available_pairs* + +``` python +if self.dp: + for pair, timeframe in self.dp.available_pairs: + print(f"available {pair}, {timeframe}") +``` + +#### *current_whitelist()* +Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. + +The strategy might look something like this: + +*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day ATR to buy and sell.* + +Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least! + +Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use. + +This is where calling `self.dp.current_whitelist()` comes in handy. + +```python +class SampleStrategy(IStrategy): + # strategy init stuff... + + ticker_interval = '5m' + + # more strategy init stuff.. + + def informative_pairs(self): + + # get access to all pairs available in whitelist. + pairs = self.dp.current_whitelist() + # Assign tf to each pair so they can be downloaded and cached for strategy. + informative_pairs = [(pair, '1d') for pair in pairs] + return informative_pairs + + def populate_indicators(self, dataframe, metadata): + # Get the informative pair + informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d') + # Get the 14 day ATR. + atr = ta.ATR(informative, timeperiod=14) + # Do other stuff +``` + +#### *get_pair_dataframe(pair, timeframe)* + +``` python +# fetch live / historical candle (OHLCV) data for the first informative pair +if self.dp: + inf_pair, inf_timeframe = self.informative_pairs()[0] + informative = self.dp.get_pair_dataframe(pair=inf_pair, + timeframe=inf_timeframe) +``` + +!!! Warning "Warning about backtesting" + Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` + for the backtesting runmode) provides the full time-range in one go, + so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). + +!!! Warning "Warning in hyperopt" + This option cannot currently be used during hyperopt. + +#### *orderbook(pair, maximum)* + +``` python +if self.dp: + if self.dp.runmode.value in ('live', 'dry_run'): + ob = self.dp.orderbook(metadata['pair'], 1) + dataframe['best_bid'] = ob['bids'][0][0] + dataframe['best_ask'] = ob['asks'][0][0] +``` + +!!! Warning + The order book is not part of the historic data which means backtesting and hyperopt will not work if this + method is used. + +*** ### Additional data (Wallets) The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. @@ -426,6 +474,7 @@ if self.wallets: - `get_used(asset)` - currently tied up balance (open orders) - `get_total(asset)` - total available balance - sum of the 2 above +*** ### Additional data (Trades) A history of Trades can be retrieved in the strategy by querying the database. diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 1df710152..984652e24 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame from freqtrade.data.history import load_pair_history +from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.state import RunMode @@ -18,9 +19,10 @@ logger = logging.getLogger(__name__) class DataProvider: - def __init__(self, config: dict, exchange: Exchange) -> None: + def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: self._config = config self._exchange = exchange + self._pairlists = pairlists def refresh(self, pairlist: List[Tuple[str, str]], @@ -116,3 +118,17 @@ class DataProvider: can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other". """ return RunMode(self._config.get('runmode', RunMode.OTHER)) + + def current_whitelist(self) -> List[str]: + """ + fetch latest available whitelist. + + Useful when you have a large whitelist and need to call each pair as an informative pair. + As available pairs does not show whitelist until after informative pairs have been cached. + :return: list of pairs in whitelist + """ + + if self._pairlists: + return self._pairlists.whitelist + else: + raise OperationalException("Dataprovider was not initialized with a pairlist provider.") diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4f4b3e3bb..73f0873e4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -71,15 +71,15 @@ class FreqtradeBot: self.wallets = Wallets(self.config, self.exchange) - self.dataprovider = DataProvider(self.config, self.exchange) + self.pairlists = PairListManager(self.exchange, self.config) + + self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass IStrategy.wallets = self.wallets - self.pairlists = PairListManager(self.exchange, self.config) - # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 2b3dda188..3e42abb95 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,8 +1,11 @@ from unittest.mock import MagicMock from pandas import DataFrame +import pytest from freqtrade.data.dataprovider import DataProvider +from freqtrade.pairlist.pairlistmanager import PairListManager +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode from tests.conftest import get_patched_exchange @@ -64,8 +67,8 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history): assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty # Test with and without parameter - assert dp.get_pair_dataframe("UNITTEST/BTC", - ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC")) + assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)\ + .equals(dp.get_pair_dataframe("UNITTEST/BTC")) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) @@ -90,10 +93,7 @@ def test_available_pairs(mocker, default_conf, ohlcv_history): dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 - assert dp.available_pairs == [ - ("XRP/BTC", ticker_interval), - ("UNITTEST/BTC", ticker_interval), - ] + assert dp.available_pairs == [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval), ] def test_refresh(mocker, default_conf, ohlcv_history): @@ -152,3 +152,27 @@ def test_market(mocker, default_conf, markets): res = dp.market('UNITTEST/BTC') assert res is None + + +def test_current_whitelist(mocker, default_conf, tickers): + # patch default conf to volumepairlist + default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5} + + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + get_tickers=tickers) + exchange = get_patched_exchange(mocker, default_conf) + + pairlist = PairListManager(exchange, default_conf) + dp = DataProvider(default_conf, exchange, pairlist) + + # Simulate volumepairs from exchange. + pairlist.refresh_pairlist() + + assert dp.current_whitelist() == pairlist._whitelist + # The identity of the 2 lists should be identical + assert dp.current_whitelist() is pairlist._whitelist + + with pytest.raises(OperationalException): + dp = DataProvider(default_conf, exchange) + dp.current_whitelist()