docs/configuration.md

@@ -222,6 +222,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange  and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation) <br> **Datatype:** Dict
 | `exchange.enable_ws` | Enable the usage of Websockets for the exchange. <br>[More information](#consuming-exchange-websockets).<br>*Defaults to `true`.* <br> **Datatype:** Boolean
 | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded. <br>*Defaults to `60` minutes.* <br> **Datatype:** Positive Integer
+| `exchange.skip_pair_validation` | Skip pairlist validation on startup.<br>*Defaults to `false`*<br> **Datatype:** Boolean
 | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.<br>*Defaults to `false`*<br> **Datatype:** Boolean
 | `exchange.unknown_fee_rate` | Fallback value to use when calculating trading fees. This can be useful for exchanges which have fees in non-tradable currencies. The value provided here will be multiplied with the "fee cost".<br>*Defaults to `None`<br> **Datatype:** float
 | `exchange.log_responses` | Log relevant exchange responses. For debug mode only - use with care.<br>*Defaults to `false`*<br> **Datatype:** Boolean

docs/developer.md

@@ -205,7 +205,7 @@ This is called with each iteration of the bot (only if the Pairlist Handler is a
 
 It must return the resulting pairlist (which may then be passed into the chain of Pairlist Handlers).
 
-Validations are optional, the parent class exposes a `verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the end-result is not shorter than expected.
+Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the end-result is not shorter than expected.
 
 #### filter_pairlist
 
@@ -219,7 +219,7 @@ The default implementation in the base class simply calls the `_validate_pair()`
 
 If overridden, it must return the resulting pairlist (which may then be passed into the next Pairlist Handler in the chain).
 
-Validations are optional, the parent class exposes a `verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the end result is not shorter than expected.
+Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the end result is not shorter than expected.
 
 In `VolumePairList`, this implements different methods of sorting, does early validation so only the expected number of pairs is returned.
 

docs/includes/pairlists.md

@@ -55,6 +55,7 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis
 By default, only currently enabled pairs are allowed.
 To skip pair validation against active markets, set `"allow_inactive": true` within the `StaticPairList` configuration.
 This can be useful for backtesting expired pairs (like quarterly spot-markets).
+This option must be configured along with `exchange.skip_pair_validation` in the exchange configuration.
 
 When used in a "follow-up" position (e.g. after VolumePairlist), all pairs in `'pair_whitelist'` will be added to the end of the pairlist.
 

freqtrade/data/history/history_utils.py

@@ -610,6 +610,9 @@ def download_data_main(config: Config) -> None:
     if "timeframes" not in config:
         config["timeframes"] = DL_DATA_TIMEFRAMES
 
+    # Manual validations of relevant settings
+    if not config["exchange"].get("skip_pair_validation", False):
+        exchange.validate_pairs(expanded_pairs)
     logger.info(
         f"About to download pairs: {expanded_pairs}, "
         f"intervals: {config['timeframes']} to {config['datadir']}"

freqtrade/exchange/exchange.py

@@ -104,6 +104,7 @@ from freqtrade.misc import (
     file_load_json,
     safe_value_fallback2,
 )
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.util import dt_from_ts, dt_now
 from freqtrade.util.datetime_helpers import dt_humanize_delta, dt_ts, format_ms_time
 from freqtrade.util.periodic_cache import PeriodicCache
@@ -330,6 +331,8 @@ class Exchange:
 
         # Check if all pairs are available
         self.validate_stakecurrency(config["stake_currency"])
+        if not config["exchange"].get("skip_pair_validation"):
+            self.validate_pairs(config["exchange"]["pair_whitelist"])
         self.validate_ordertypes(config.get("order_types", {}))
         self.validate_order_time_in_force(config.get("order_time_in_force", {}))
         self.validate_trading_mode_and_margin_mode(self.trading_mode, self.margin_mode)
@@ -699,6 +702,54 @@ class Exchange:
                 f"Available currencies are: {', '.join(quote_currencies)}"
             )
 
+    def validate_pairs(self, pairs: List[str]) -> None:
+        """
+        Checks if all given pairs are tradable on the current exchange.
+        :param pairs: list of pairs
+        :raise: OperationalException if one pair is not available
+        :return: None
+        """
+
+        if not self.markets:
+            logger.warning("Unable to validate pairs (assuming they are correct).")
+            return
+        extended_pairs = expand_pairlist(pairs, list(self.markets), keep_invalid=True)
+        invalid_pairs = []
+        for pair in extended_pairs:
+            # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
+            if self.markets and pair not in self.markets:
+                raise OperationalException(
+                    f"Pair {pair} is not available on {self.name} {self.trading_mode}. "
+                    f"Please remove {pair} from your whitelist."
+                )
+
+                # From ccxt Documentation:
+                # markets.info: An associative array of non-common market properties,
+                # including fees, rates, limits and other general market information.
+                # The internal info array is different for each particular market,
+                # its contents depend on the exchange.
+                # It can also be a string or similar ... so we need to verify that first.
+            elif isinstance(self.markets[pair].get("info"), dict) and self.markets[pair].get(
+                "info", {}
+            ).get("prohibitedIn", False):
+                # Warn users about restricted pairs in whitelist.
+                # We cannot determine reliably if Users are affected.
+                logger.warning(
+                    f"Pair {pair} is restricted for some users on this exchange."
+                    f"Please check if you are impacted by this restriction "
+                    f"on the exchange and eventually remove {pair} from your whitelist."
+                )
+            if (
+                self._config["stake_currency"]
+                and self.get_pair_quote_currency(pair) != self._config["stake_currency"]
+            ):
+                invalid_pairs.append(pair)
+        if invalid_pairs:
+            raise OperationalException(
+                f"Stake-currency '{self._config['stake_currency']}' not compatible with "
+                f"pair-whitelist. Please remove the following pairs: {invalid_pairs}"
+            )
+
     def get_valid_pair_combination(self, curr_1: str, curr_2: str) -> str:
         """
         Get valid pair combination of curr_1 and curr_2 by trying both combinations.

freqtrade/plugins/pairlist/StaticPairList.py

@@ -61,15 +61,14 @@ class StaticPairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers). May be cached.
         :return: List of pairs
         """
-        wl = self.verify_whitelist(
-            self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True
-        )
         if self._allow_inactive:
-            return wl
+            return self.verify_whitelist(
+                self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True
+            )
         else:
-            # Avoid implicit filtering of "verify_whitelist" to keep
-            # proper warnings in the log
-            return self._whitelist_for_active_markets(wl)
+            return self._whitelist_for_active_markets(
+                self.verify_whitelist(self._config["exchange"]["pair_whitelist"], logger.info)
+            )
 
     def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]:
         """

freqtrade/plugins/pairlist/pairlist_helpers.py

@@ -28,7 +28,6 @@ def expand_pairlist(
             except re.error as err:
                 raise ValueError(f"Wildcard error in {pair_wc}, {err}")
 
-        # Remove wildcard pairs that didn't have a match.
         result = [element for element in result if re.fullmatch(r"^[A-Za-z0-9:/-]+$", element)]
 
     else:

tests/exchange/test_exchange.py

@@ -255,6 +255,7 @@ def test_init_exception(default_conf, mocker):
 def test_exchange_resolver(default_conf, mocker, caplog):
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=MagicMock()))
     mocker.patch(f"{EXMS}._load_async_markets")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_timeframes")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
     mocker.patch(f"{EXMS}.validate_pricing")
@@ -554,6 +555,7 @@ def test_get_min_pair_stake_amount_real_data(mocker, default_conf) -> None:
 
 def test__load_async_markets(default_conf, mocker, caplog):
     mocker.patch(f"{EXMS}._init_ccxt")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_timeframes")
     mocker.patch(f"{EXMS}.reload_markets")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
@@ -582,6 +584,7 @@ def test__load_markets(default_conf, mocker, caplog):
     api_mock = MagicMock()
     api_mock.load_markets = get_mock_coro(side_effect=ccxt.BaseError("SomeError"))
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_timeframes")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
     mocker.patch(f"{EXMS}.validate_pricing")
@@ -681,6 +684,7 @@ def test_validate_stakecurrency(default_conf, stake_currency, mocker, caplog):
         }
     )
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_timeframes")
     mocker.patch(f"{EXMS}.validate_pricing")
     Exchange(default_conf)
@@ -698,6 +702,7 @@ def test_validate_stakecurrency_error(default_conf, mocker, caplog):
         }
     )
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_timeframes")
     with pytest.raises(
         ConfigurationError,
@@ -750,6 +755,147 @@ def test_get_pair_base_currency(default_conf, mocker, pair, expected):
     assert ex.get_pair_base_currency(pair) == expected
 
 
+def test_validate_pairs(default_conf, mocker):
+    api_mock = MagicMock()
+    id_mock = PropertyMock(return_value="test_exchange")
+    type(api_mock).id = id_mock
+
+    mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
+    mocker.patch(f"{EXMS}.validate_timeframes")
+    mocker.patch(
+        f"{EXMS}._load_async_markets",
+        return_value={
+            "ETH/BTC": {"quote": "BTC"},
+            "LTC/BTC": {"quote": "BTC"},
+            "XRP/BTC": {"quote": "BTC"},
+            "NEO/BTC": {"quote": "BTC"},
+        },
+    )
+    mocker.patch(f"{EXMS}.validate_stakecurrency")
+    mocker.patch(f"{EXMS}.validate_pricing")
+    # test exchange.validate_pairs directly
+    # No assert - but this should not fail (!)
+    Exchange(default_conf)
+
+
+def test_validate_pairs_not_available(default_conf, mocker):
+    api_mock = MagicMock()
+    type(api_mock).markets = PropertyMock(
+        return_value={"XRP/BTC": {"inactive": True, "base": "XRP", "quote": "BTC"}}
+    )
+    mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
+    mocker.patch(f"{EXMS}.validate_timeframes")
+    mocker.patch(f"{EXMS}.validate_stakecurrency")
+    mocker.patch(f"{EXMS}._load_async_markets")
+
+    with pytest.raises(OperationalException, match=r"not available"):
+        Exchange(default_conf)
+
+
+def test_validate_pairs_exception(default_conf, mocker, caplog):
+    caplog.set_level(logging.INFO)
+    api_mock = MagicMock()
+    mocker.patch(f"{EXMS}.name", PropertyMock(return_value="Binance"))
+
+    type(api_mock).markets = PropertyMock(return_value={})
+    mocker.patch(f"{EXMS}._init_ccxt", api_mock)
+    mocker.patch(f"{EXMS}.validate_timeframes")
+    mocker.patch(f"{EXMS}.validate_stakecurrency")
+    mocker.patch(f"{EXMS}.validate_pricing")
+    mocker.patch(f"{EXMS}._load_async_markets")
+
+    with pytest.raises(OperationalException, match=r"Pair ETH/BTC is not available on Binance"):
+        Exchange(default_conf)
+
+    mocker.patch(f"{EXMS}.markets", PropertyMock(return_value={}))
+    Exchange(default_conf)
+    assert log_has("Unable to validate pairs (assuming they are correct).", caplog)
+
+
+def test_validate_pairs_restricted(default_conf, mocker, caplog):
+    api_mock = MagicMock()
+    type(api_mock).load_markets = get_mock_coro(
+        return_value={
+            "ETH/BTC": {"quote": "BTC"},
+            "LTC/BTC": {"quote": "BTC"},
+            "XRP/BTC": {"quote": "BTC", "info": {"prohibitedIn": ["US"]}},
+            "NEO/BTC": {"quote": "BTC", "info": "TestString"},  # info can also be a string ...
+        }
+    )
+    mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
+    mocker.patch(f"{EXMS}.validate_timeframes")
+    mocker.patch(f"{EXMS}.validate_pricing")
+    mocker.patch(f"{EXMS}.validate_stakecurrency")
+
+    Exchange(default_conf)
+    assert log_has(
+        "Pair XRP/BTC is restricted for some users on this exchange."
+        "Please check if you are impacted by this restriction "
+        "on the exchange and eventually remove XRP/BTC from your whitelist.",
+        caplog,
+    )
+
+
+def test_validate_pairs_stakecompatibility(default_conf, mocker):
+    api_mock = MagicMock()
+    type(api_mock).load_markets = get_mock_coro(
+        return_value={
+            "ETH/BTC": {"quote": "BTC"},
+            "LTC/BTC": {"quote": "BTC"},
+            "XRP/BTC": {"quote": "BTC"},
+            "NEO/BTC": {"quote": "BTC"},
+            "HELLO-WORLD": {"quote": "BTC"},
+        }
+    )
+    mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
+    mocker.patch(f"{EXMS}.validate_timeframes")
+    mocker.patch(f"{EXMS}.validate_stakecurrency")
+    mocker.patch(f"{EXMS}.validate_pricing")
+
+    Exchange(default_conf)
+
+
+def test_validate_pairs_stakecompatibility_downloaddata(default_conf, mocker):
+    api_mock = MagicMock()
+    default_conf["stake_currency"] = ""
+    type(api_mock).load_markets = get_mock_coro(
+        return_value={
+            "ETH/BTC": {"quote": "BTC"},
+            "LTC/BTC": {"quote": "BTC"},
+            "XRP/BTC": {"quote": "BTC"},
+            "NEO/BTC": {"quote": "BTC"},
+            "HELLO-WORLD": {"quote": "BTC"},
+        }
+    )
+    mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
+    mocker.patch(f"{EXMS}.validate_timeframes")
+    mocker.patch(f"{EXMS}.validate_stakecurrency")
+    mocker.patch(f"{EXMS}.validate_pricing")
+
+    Exchange(default_conf)
+    assert type(api_mock).load_markets.call_count == 1
+
+
+def test_validate_pairs_stakecompatibility_fail(default_conf, mocker):
+    default_conf["exchange"]["pair_whitelist"].append("HELLO-WORLD")
+    api_mock = MagicMock()
+    type(api_mock).load_markets = get_mock_coro(
+        return_value={
+            "ETH/BTC": {"quote": "BTC"},
+            "LTC/BTC": {"quote": "BTC"},
+            "XRP/BTC": {"quote": "BTC"},
+            "NEO/BTC": {"quote": "BTC"},
+            "HELLO-WORLD": {"quote": "USDT"},
+        }
+    )
+    mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
+    mocker.patch(f"{EXMS}.validate_timeframes")
+    mocker.patch(f"{EXMS}.validate_stakecurrency")
+
+    with pytest.raises(OperationalException, match=r"Stake-currency 'BTC' not compatible with.*"):
+        Exchange(default_conf)
+
+
 @pytest.mark.parametrize("timeframe", [("5m"), ("1m"), ("15m"), ("1h")])
 def test_validate_timeframes(default_conf, mocker, timeframe):
     default_conf["timeframe"] = timeframe
@@ -761,6 +907,7 @@ def test_validate_timeframes(default_conf, mocker, timeframe):
 
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
     mocker.patch(f"{EXMS}.reload_markets")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
     mocker.patch(f"{EXMS}.validate_pricing")
     Exchange(default_conf)
@@ -778,6 +925,7 @@ def test_validate_timeframes_failed(default_conf, mocker):
 
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
     mocker.patch(f"{EXMS}.reload_markets")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
     mocker.patch(f"{EXMS}.validate_pricing")
     with pytest.raises(
@@ -807,6 +955,7 @@ def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker):
 
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
     mocker.patch(f"{EXMS}.reload_markets")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
     with pytest.raises(
         OperationalException,
@@ -828,6 +977,7 @@ def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker):
 
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
     mocker.patch(f"{EXMS}.reload_markets")
+    mocker.patch(f"{EXMS}.validate_pairs", MagicMock())
     mocker.patch(f"{EXMS}.validate_stakecurrency")
     with pytest.raises(
         OperationalException,
@@ -849,6 +999,7 @@ def test_validate_timeframes_not_in_config(default_conf, mocker):
 
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
     mocker.patch(f"{EXMS}.reload_markets")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
     mocker.patch(f"{EXMS}.validate_pricing")
     mocker.patch(f"{EXMS}.validate_required_startup_candles")
@@ -865,6 +1016,7 @@ def test_validate_pricing(default_conf, mocker):
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
     mocker.patch(f"{EXMS}.reload_markets")
     mocker.patch(f"{EXMS}.validate_trading_mode_and_margin_mode")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_timeframes")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
     mocker.patch(f"{EXMS}.name", "Binance")
@@ -899,6 +1051,7 @@ def test_validate_ordertypes(default_conf, mocker):
     type(api_mock).has = PropertyMock(return_value={"createMarketOrder": True})
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
     mocker.patch(f"{EXMS}.reload_markets")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_timeframes")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
     mocker.patch(f"{EXMS}.validate_pricing")
@@ -957,6 +1110,7 @@ def test_validate_ordertypes_stop_advanced(default_conf, mocker, exchange_name,
     type(api_mock).has = PropertyMock(return_value={"createMarketOrder": True})
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
     mocker.patch(f"{EXMS}.reload_markets")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_timeframes")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
     mocker.patch(f"{EXMS}.validate_pricing")
@@ -981,6 +1135,7 @@ def test_validate_order_types_not_in_config(default_conf, mocker):
     api_mock = MagicMock()
     mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
     mocker.patch(f"{EXMS}.reload_markets")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_timeframes")
     mocker.patch(f"{EXMS}.validate_pricing")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
@@ -996,6 +1151,7 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog):
     mocker.patch(f"{EXMS}._init_ccxt", api_mock)
     mocker.patch(f"{EXMS}.validate_timeframes")
     mocker.patch(f"{EXMS}._load_async_markets")
+    mocker.patch(f"{EXMS}.validate_pairs")
     mocker.patch(f"{EXMS}.validate_pricing")
     mocker.patch(f"{EXMS}.validate_stakecurrency")
 
@@ -4029,6 +4185,7 @@ def test_merge_ft_has_dict(default_conf, mocker):
         EXMS,
         _init_ccxt=MagicMock(return_value=MagicMock()),
         _load_async_markets=MagicMock(),
+        validate_pairs=MagicMock(),
         validate_timeframes=MagicMock(),
         validate_stakecurrency=MagicMock(),
         validate_pricing=MagicMock(),
@@ -4063,6 +4220,7 @@ def test_get_valid_pair_combination(default_conf, mocker, markets):
         EXMS,
         _init_ccxt=MagicMock(return_value=MagicMock()),
         _load_async_markets=MagicMock(),
+        validate_pairs=MagicMock(),
         validate_timeframes=MagicMock(),
         validate_pricing=MagicMock(),
         markets=PropertyMock(return_value=markets),
@@ -4342,6 +4500,7 @@ def test_get_markets(
         EXMS,
         _init_ccxt=MagicMock(return_value=MagicMock()),
         _load_async_markets=MagicMock(),
+        validate_pairs=MagicMock(),
         validate_timeframes=MagicMock(),
         validate_pricing=MagicMock(),
         markets=PropertyMock(return_value=markets_static),

tests/freqtradebot/test_freqtradebot.py

@@ -2204,6 +2204,7 @@ def test_manage_open_orders_buy_exception(
     patch_exchange(mocker)
     mocker.patch.multiple(
         EXMS,
+        validate_pairs=MagicMock(),
         fetch_ticker=ticker_usdt,
         fetch_order=MagicMock(side_effect=ExchangeError),
         cancel_order=cancel_order_mock,