Merge pull request #1661 from iuvbio/validate_whitelist

validate whitelist vs. validate pairs

freqtrade/exchange/exchange.py

@@ -239,13 +239,9 @@ class Exchange(object):
             logger.warning('Unable to validate pairs (assuming they are correct).')
         #     return
 
-        stake_cur = self._config['stake_currency']
         for pair in pairs:
             # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
             # TODO: add a support for having coins in BTC/USDT format
-            if not pair.endswith(stake_cur):
-                raise OperationalException(
-                    f'Pair {pair} not compatible with stake_currency: {stake_cur}')
             if self.markets and pair not in self.markets:
                 raise OperationalException(
                     f'Pair {pair} is not available on {self.name}. '

freqtrade/pairlist/IPairList.py

@@ -60,34 +60,27 @@ class IPairList(ABC):
     def _validate_whitelist(self, whitelist: List[str]) -> List[str]:
         """
         Check available markets and remove pair from whitelist if necessary
-        :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to
+        :param whitelist: the sorted list of pairs the user might want to trade
-        trade
+        :return: the list of pairs the user wants to trade without those unavailable or
-        :return: the list of pairs the user wants to trade without the one unavailable or
         black_listed
         """
-        sanitized_whitelist = whitelist
         markets = self._freqtrade.exchange.markets
 
-        # Filter to markets in stake currency
+        sanitized_whitelist = set()
-        markets = [markets[pair] for pair in markets if
+        for pair in whitelist:
-                   markets[pair]['quote'] == self._config['stake_currency']]
-        known_pairs = set()
-
-        # TODO: we should loop over whitelist instead of all markets
-        for market in markets:
-            pair = market['symbol']
             # pair is not in the generated dynamic market, or in the blacklist ... ignore it
-            if pair not in whitelist or pair in self.blacklist:
+            if (pair in self.blacklist or pair not in markets
+                    or not pair.endswith(self._config['stake_currency'])):
+                logger.warning(f"Pair {pair} is not compatible with exchange "
+                               f"{self._freqtrade.exchange.name} or contained in "
+                               f"your blacklist. Removing it from whitelist..")
                 continue
-            # else the pair is valid
+            # Check if market is active
-            known_pairs.add(pair)
+            market = markets[pair]
-            # Market is not active
             if not market['active']:
-                sanitized_whitelist.remove(pair)
+                logger.info(f"Ignoring {pair} from whitelist. Market is not active.")
-                logger.info(
+                continue
-                    'Ignoring %s from whitelist. Market is not active.',
+            sanitized_whitelist.add(pair)
-                    pair
-                )
 
         # We need to remove pairs that are unknown
-        return [x for x in sanitized_whitelist if x in known_pairs]
+        return list(sanitized_whitelist)

freqtrade/tests/exchange/test_exchange.py

@@ -305,19 +305,6 @@ def test_validate_pairs_not_available(default_conf, mocker):
         Exchange(default_conf)
 
 
-def test_validate_pairs_not_compatible(default_conf, mocker):
-    api_mock = MagicMock()
-    type(api_mock).markets = PropertyMock(return_value={
-        'ETH/BTC': '', 'TKN/BTC': '', 'TRST/BTC': '', 'SWT/BTC': '', 'BCC/BTC': ''
-    })
-    default_conf['stake_currency'] = 'ETH'
-    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
-    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
-    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
-    with pytest.raises(OperationalException, match=r'not compatible'):
-        Exchange(default_conf)
-
-
 def test_validate_pairs_exception(default_conf, mocker, caplog):
     caplog.set_level(logging.INFO)
     api_mock = MagicMock()
@@ -337,22 +324,6 @@ def test_validate_pairs_exception(default_conf, mocker, caplog):
                    caplog.record_tuples)
 
 
-def test_validate_pairs_stake_exception(default_conf, mocker, caplog):
-    caplog.set_level(logging.INFO)
-    default_conf['stake_currency'] = 'ETH'
-    api_mock = MagicMock()
-    api_mock.name = MagicMock(return_value='binance')
-    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
-    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
-    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
-
-    with pytest.raises(
-        OperationalException,
-        match=r'Pair ETH/BTC not compatible with stake_currency: ETH'
-    ):
-        Exchange(default_conf)
-
-
 def test_validate_timeframes(default_conf, mocker):
     default_conf["ticker_interval"] = "5m"
     api_mock = MagicMock()

freqtrade/tests/pairlist/test_pairlist.py

@@ -107,7 +107,16 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     assert set(whitelist) == set(pairslist)
 
 
-def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) -> None:
+@pytest.mark.parametrize("precision_filter,base_currency,key,whitelist_result", [
+    (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'BTT/BTC']),
+    (False, "BTC", "bidVolume", ['BTT/BTC', 'TKN/BTC', 'ETH/BTC']),
+    (False, "USDT", "quoteVolume", ['ETH/USDT', 'LTC/USDT']),
+    (False, "ETH", "quoteVolume", []),  # this replaces tests that were removed from test_exchange
+    (True, "BTC", "quoteVolume", ["ETH/BTC", "TKN/BTC"]),
+    (True, "BTC", "bidVolume", ["TKN/BTC", "ETH/BTC"])
+])
+def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers, base_currency, key,
+                                      whitelist_result, precision_filter) -> None:
     whitelist_conf['pairlist']['method'] = 'VolumePairList'
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
     freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
@@ -115,32 +124,10 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers)
     mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)
     mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, p, r: round(r, 8))
 
-    # Test to retrieved BTC sorted on quoteVolume (default)
+    freqtrade.pairlists._precision_filter = precision_filter
-    whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume')
+    freqtrade.config['stake_currency'] = base_currency
-    assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BTT/BTC']
+    whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key)
-
+    assert whitelist == whitelist_result
-    # Test to retrieve BTC sorted on bidVolume
-    whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='bidVolume')
-    assert whitelist == ['BTT/BTC', 'TKN/BTC', 'ETH/BTC']
-
-    # Test with USDT sorted on quoteVolume (default)
-    freqtrade.config['stake_currency'] = 'USDT'  # this has to be set, otherwise markets are removed
-    whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='USDT', key='quoteVolume')
-    assert whitelist == ['ETH/USDT', 'LTC/USDT']
-
-    # Test with ETH (our fixture does not have ETH, so result should be empty)
-    freqtrade.config['stake_currency'] = 'ETH'
-    whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='ETH', key='quoteVolume')
-    assert whitelist == []
-
-    freqtrade.pairlists._precision_filter = True
-    freqtrade.config['stake_currency'] = 'BTC'
-    # Retest First 2 test-cases to make sure BTT is not in it (too low priced)
-    whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume')
-    assert whitelist == ['ETH/BTC', 'TKN/BTC']
-
-    whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='bidVolume')
-    assert whitelist == ['TKN/BTC', 'ETH/BTC']
 
 
 def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None:
@@ -166,17 +153,24 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist):
     assert isinstance(freqtrade.pairlists.whitelist, list)
     assert isinstance(freqtrade.pairlists.blacklist, list)
 
-    whitelist = ['ETH/BTC', 'TKN/BTC']
+
+@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS)
+@pytest.mark.parametrize("whitelist,log_message", [
+    (['ETH/BTC', 'TKN/BTC'], ""),
+    (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"),  # TRX/ETH wrong stake
+    (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"),  # BCH/BTC not available
+    (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "is not compatible with exchange"),  # BLK/BTC in blacklist
+    (['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], "Market is not active")  # LTC/BTC is inactive
+])
+def test_validate_whitelist(mocker, whitelist_conf, markets, pairlist, whitelist, caplog,
+                            log_message):
+    whitelist_conf['pairlist']['method'] = pairlist
+    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+    caplog.clear()
+
     new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist)
 
-    assert set(whitelist) == set(new_whitelist)
+    assert set(new_whitelist) == set(['ETH/BTC', 'TKN/BTC'])
-
+    assert log_message in caplog.text
-    whitelist = ['ETH/BTC', 'TKN/BTC', 'TRX/ETH']
-    new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist)
-    # TRX/ETH was removed
-    assert set(['ETH/BTC', 'TKN/BTC']) == set(new_whitelist)
-
-    whitelist = ['ETH/BTC', 'TKN/BTC', 'BLK/BTC']
-    new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist)
-    # BLK/BTC is in blacklist ...
-    assert set(['ETH/BTC', 'TKN/BTC']) == set(new_whitelist)