diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 5fda038bd..1babeca1f 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -184,6 +184,8 @@ The RemotePairList is defined in the pairlists section of the configuration sett "pairlists": [ { "method": "RemotePairList", + "mode": "whitelist", + "processing_mode": "filter", "pairlist_url": "https://example.com/pairlist", "number_assets": 10, "refresh_period": 1800, @@ -194,6 +196,14 @@ The RemotePairList is defined in the pairlists section of the configuration sett ] ``` +The optional `mode` option specifies if the pairlist should be used as a `blacklist` or as a `whitelist`. The default value is "whitelist". + +The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append". + +In "filter" mode, the retrieved pairlist is used as a filter. Only the pairs present in both the original pairlist and the retrieved pairlist are included in the final pairlist. Other pairs are filtered out. + +In "append" mode, the retrieved pairlist is added to the original pairlist. All pairs from both lists are included in the final pairlist without any filtering. + The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist. The user is responsible for providing a server or local file that returns a JSON object with the following structure: diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 372f9a593..66b7d9496 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -16,6 +16,7 @@ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist logger = logging.getLogger(__name__) @@ -40,6 +41,8 @@ class RemotePairList(IPairList): '`pairlist_url` not specified. Please check your configuration ' 'for "pairlist.config.pairlist_url"') + self._mode = self._pairlistconfig.get('mode', 'whitelist') + self._processing_mode = self._pairlistconfig.get('processing_mode', 'filter') self._number_pairs = self._pairlistconfig['number_assets'] self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800) self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True) @@ -50,6 +53,21 @@ class RemotePairList(IPairList): self._init_done = False self._last_pairlist: List[Any] = list() + if self._mode not in ['whitelist', 'blacklist']: + raise OperationalException( + '`mode` not configured correctly. Supported Modes ' + 'are "whitelist","blacklist"') + + if self._processing_mode not in ['filter', 'append']: + raise OperationalException( + '`processing_mode` not configured correctly. Supported Modes ' + 'are "filter","append"') + + if self._pairlist_pos == 0 and self._mode == 'blacklist': + raise OperationalException( + 'A `blacklist` mode RemotePairList can not be on the first ' + 'position of your pairlist.') + @property def needstickers(self) -> bool: """ @@ -67,23 +85,37 @@ class RemotePairList(IPairList): @staticmethod def description() -> str: - return "Retrieve pairs from a remote API." + return "Retrieve pairs from a remote API or local file." @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { - "number_assets": { - "type": "number", - "default": 0, - "description": "Number of assets", - "help": "Number of assets to use from the pairlist.", - }, "pairlist_url": { "type": "string", "default": "", "description": "URL to fetch pairlist from", "help": "URL to fetch pairlist from", }, + "number_assets": { + "type": "number", + "default": 30, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist.", + }, + "mode": { + "type": "option", + "default": "whitelist", + "options": ["whitelist", "blacklist"], + "description": "Pairlist mode", + "help": "Should this pairlist operate as a whitelist or blacklist?", + }, + "processing_mode": { + "type": "option", + "default": "filter", + "options": ["filter", "append"], + "description": "Processing mode", + "help": "Append pairs to incomming pairlist or filter them?", + }, **IPairList.refresh_period_parameter(), "keep_pairlist_on_failure": { "type": "boolean", @@ -223,6 +255,7 @@ class RemotePairList(IPairList): self.log_once(f"Fetched pairs: {pairlist}", logger.debug) + pairlist = expand_pairlist(pairlist, list(self._exchange.get_markets().keys())) pairlist = self._whitelist_for_active_markets(pairlist) pairlist = pairlist[:self._number_pairs] @@ -250,6 +283,23 @@ class RemotePairList(IPairList): :return: new whitelist """ rpl_pairlist = self.gen_pairlist(tickers) - merged_list = pairlist + rpl_pairlist - merged_list = sorted(set(merged_list), key=merged_list.index) + merged_list = [] + filtered = [] + + if self._mode == "whitelist": + if self._processing_mode == "filter": + merged_list = [pair for pair in pairlist if pair in rpl_pairlist] + elif self._processing_mode == "append": + merged_list = pairlist + rpl_pairlist + merged_list = sorted(set(merged_list), key=merged_list.index) + else: + for pair in pairlist: + if pair not in rpl_pairlist: + merged_list.append(pair) + else: + filtered.append(pair) + if filtered: + self.log_once(f"Blacklist - Filtered out pairs: {filtered}", logger.info) + + merged_list = merged_list[:self._number_pairs] return merged_list diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index bc8fe84f1..cbbf1e6da 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1200,6 +1200,10 @@ def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplo "[{'ProducerPairList': 'ProducerPairList - default'}]", None ), + ({"method": "RemotePairList", "number_assets": 10, "pairlist_url": "https://example.com"}, + "[{'RemotePairList': 'RemotePairList - 10 pairs from RemotePairlist.'}]", + None + ), ]) def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig, desc_expected, exception_expected): diff --git a/tests/plugins/test_remotepairlist.py b/tests/plugins/test_remotepairlist.py index ac1d1f5ed..5e6f5cbf1 100644 --- a/tests/plugins/test_remotepairlist.py +++ b/tests/plugins/test_remotepairlist.py @@ -1,5 +1,5 @@ import json -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import pytest import requests @@ -7,7 +7,7 @@ import requests from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.RemotePairList import RemotePairList from freqtrade.plugins.pairlistmanager import PairListManager -from tests.conftest import get_patched_exchange, get_patched_freqtradebot, log_has +from tests.conftest import EXMS, get_patched_exchange, get_patched_freqtradebot, log_has @pytest.fixture(scope="function") @@ -16,11 +16,12 @@ def rpl_config(default_conf): default_conf['exchange']['pair_whitelist'] = [ 'ETH/USDT', - 'BTC/USDT', + 'XRP/USDT', ] default_conf['exchange']['pair_blacklist'] = [ 'BLK/USDT' ] + return default_conf @@ -183,3 +184,159 @@ def test_fetch_pairlist_mock_response_valid(mocker, rpl_config): assert pairs == ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"] assert time_elapsed == 0.4 assert remote_pairlist._refresh_period == 60 + + +def test_remote_pairlist_init_wrong_mode(mocker, rpl_config): + rpl_config['pairlists'] = [ + { + "method": "RemotePairList", + "mode": "blacklis", + "number_assets": 20, + "pairlist_url": "http://example.com/pairlist", + "keep_pairlist_on_failure": True, + } + ] + + with pytest.raises( + OperationalException, + match=r'`mode` not configured correctly. Supported Modes are "whitelist","blacklist"' + ): + get_patched_freqtradebot(mocker, rpl_config) + + rpl_config['pairlists'] = [ + { + "method": "RemotePairList", + "mode": "blacklist", + "number_assets": 20, + "pairlist_url": "http://example.com/pairlist", + "keep_pairlist_on_failure": True, + } + ] + + with pytest.raises( + OperationalException, + match=r'A `blacklist` mode RemotePairList can not be.*first.*' + ): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_remote_pairlist_init_wrong_proc_mode(mocker, rpl_config): + rpl_config['pairlists'] = [ + { + "method": "RemotePairList", + "processing_mode": "filler", + "mode": "whitelist", + "number_assets": 20, + "pairlist_url": "http://example.com/pairlist", + "keep_pairlist_on_failure": True, + } + ] + + get_patched_exchange(mocker, rpl_config) + with pytest.raises( + OperationalException, + match=r'`processing_mode` not configured correctly. Supported Modes are "filter","append"' + ): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_remote_pairlist_blacklist(mocker, rpl_config, caplog, markets, tickers): + + mock_response = MagicMock() + + mock_response.json.return_value = { + "pairs": ["XRP/USDT"], + "refresh_period": 60 + } + + mock_response.headers = { + "content-type": "application/json" + } + + rpl_config['pairlists'] = [ + { + "method": "StaticPairList", + }, + { + "method": "RemotePairList", + "mode": "blacklist", + "pairlist_url": "http://example.com/pairlist", + "number_assets": 3 + } + ] + + mocker.patch.multiple(EXMS, + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get", + return_value=mock_response) + + exchange = get_patched_exchange(mocker, rpl_config) + + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, + rpl_config["pairlists"][1], 1) + + pairs, time_elapsed = remote_pairlist.fetch_pairlist() + + assert pairs == ["XRP/USDT"] + + whitelist = remote_pairlist.filter_pairlist(rpl_config['exchange']['pair_whitelist'], {}) + assert whitelist == ["ETH/USDT"] + + assert log_has(f"Blacklist - Filtered out pairs: {pairs}", caplog) + + +@pytest.mark.parametrize("processing_mode", ["filter", "append"]) +def test_remote_pairlist_whitelist(mocker, rpl_config, processing_mode, markets, tickers): + + mock_response = MagicMock() + + mock_response.json.return_value = { + "pairs": ["XRP/USDT"], + "refresh_period": 60 + } + + mock_response.headers = { + "content-type": "application/json" + } + + rpl_config['pairlists'] = [ + { + "method": "StaticPairList", + }, + { + "method": "RemotePairList", + "mode": "whitelist", + "processing_mode": processing_mode, + "pairlist_url": "http://example.com/pairlist", + "number_assets": 3 + } + ] + + mocker.patch.multiple(EXMS, + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get", + return_value=mock_response) + + exchange = get_patched_exchange(mocker, rpl_config) + + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, + rpl_config["pairlists"][1], 1) + + pairs, time_elapsed = remote_pairlist.fetch_pairlist() + + assert pairs == ["XRP/USDT"] + + whitelist = remote_pairlist.filter_pairlist(rpl_config['exchange']['pair_whitelist'], {}) + assert whitelist == (["XRP/USDT"] if processing_mode == "filter" else ['ETH/USDT', 'XRP/USDT'])