Compare commits

...

9 Commits

Author SHA1 Message Date
Matthias
ae41ab101a docs: remove skip_pair_validation - it's no longer used.
Some checks are pending
Build Documentation / Deploy Docs through mike (push) Waiting to run
2024-09-15 11:28:57 +02:00
Matthias
f4881e7c6f tests: Adjust tests for removed validate_pairlist functionality 2024-09-15 11:28:57 +02:00
Matthias
94ef4380d4 chore: remove validate_pairs from exchange class
Invalid pairs were filtered out before this was called in most cases.
in cases where it's not - regular pairlist-filtering provides proper warnings.
2024-09-15 11:28:57 +02:00
Matthias
7ebe1b8c14 chore: remove pointless validation
pairs are validated through expand_pairlist.
If they're not in markets, they'll no longer be in the
pairlist once this function function is hit.
2024-09-15 11:02:49 +02:00
Matthias
79020bba28 chore: Remove "prohibitedIn" check
it's only been used for bitrex, which does no longer exist.
apparently this was forgotten when decomissioning bittrex.
2024-09-15 10:49:26 +02:00
Matthias
95c250ebcc chore: add explaining comment 2024-09-15 10:37:28 +02:00
Matthias
bfb14614cc chore: enhance change with comment 2024-09-15 09:48:44 +02:00
Matthias
12299d4810 feat: staticPairlist to warn for invalid pairs
Warnings about invalid pairs were "covered" by the implicit
filtering of `expand_pairlist()`
2024-09-15 09:46:47 +02:00
Matthias
c67a9d4e84 docs: update pairlist creation docs 2024-09-15 09:29:45 +02:00
9 changed files with 10 additions and 224 deletions

View File

@ -222,7 +222,6 @@ Mandatory parameters are marked as **Required**, which means that they are requi
| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation) <br> **Datatype:** Dict
| `exchange.enable_ws` | Enable the usage of Websockets for the exchange. <br>[More information](#consuming-exchange-websockets).<br>*Defaults to `true`.* <br> **Datatype:** Boolean
| `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded. <br>*Defaults to `60` minutes.* <br> **Datatype:** Positive Integer
| `exchange.skip_pair_validation` | Skip pairlist validation on startup.<br>*Defaults to `false`*<br> **Datatype:** Boolean
| `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.<br>*Defaults to `false`*<br> **Datatype:** Boolean
| `exchange.unknown_fee_rate` | Fallback value to use when calculating trading fees. This can be useful for exchanges which have fees in non-tradable currencies. The value provided here will be multiplied with the "fee cost".<br>*Defaults to `None`<br> **Datatype:** float
| `exchange.log_responses` | Log relevant exchange responses. For debug mode only - use with care.<br>*Defaults to `false`*<br> **Datatype:** Boolean

View File

@ -205,7 +205,7 @@ This is called with each iteration of the bot (only if the Pairlist Handler is a
It must return the resulting pairlist (which may then be passed into the chain of Pairlist Handlers).
Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the end-result is not shorter than expected.
Validations are optional, the parent class exposes a `verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the end-result is not shorter than expected.
#### filter_pairlist
@ -219,7 +219,7 @@ The default implementation in the base class simply calls the `_validate_pair()`
If overridden, it must return the resulting pairlist (which may then be passed into the next Pairlist Handler in the chain).
Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the end result is not shorter than expected.
Validations are optional, the parent class exposes a `verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the end result is not shorter than expected.
In `VolumePairList`, this implements different methods of sorting, does early validation so only the expected number of pairs is returned.

View File

@ -55,7 +55,6 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis
By default, only currently enabled pairs are allowed.
To skip pair validation against active markets, set `"allow_inactive": true` within the `StaticPairList` configuration.
This can be useful for backtesting expired pairs (like quarterly spot-markets).
This option must be configured along with `exchange.skip_pair_validation` in the exchange configuration.
When used in a "follow-up" position (e.g. after VolumePairlist), all pairs in `'pair_whitelist'` will be added to the end of the pairlist.

View File

@ -610,9 +610,6 @@ def download_data_main(config: Config) -> None:
if "timeframes" not in config:
config["timeframes"] = DL_DATA_TIMEFRAMES
# Manual validations of relevant settings
if not config["exchange"].get("skip_pair_validation", False):
exchange.validate_pairs(expanded_pairs)
logger.info(
f"About to download pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}"

View File

@ -104,7 +104,6 @@ from freqtrade.misc import (
file_load_json,
safe_value_fallback2,
)
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.util import dt_from_ts, dt_now
from freqtrade.util.datetime_helpers import dt_humanize_delta, dt_ts, format_ms_time
from freqtrade.util.periodic_cache import PeriodicCache
@ -331,8 +330,6 @@ class Exchange:
# Check if all pairs are available
self.validate_stakecurrency(config["stake_currency"])
if not config["exchange"].get("skip_pair_validation"):
self.validate_pairs(config["exchange"]["pair_whitelist"])
self.validate_ordertypes(config.get("order_types", {}))
self.validate_order_time_in_force(config.get("order_time_in_force", {}))
self.validate_trading_mode_and_margin_mode(self.trading_mode, self.margin_mode)
@ -702,54 +699,6 @@ class Exchange:
f"Available currencies are: {', '.join(quote_currencies)}"
)
def validate_pairs(self, pairs: List[str]) -> None:
"""
Checks if all given pairs are tradable on the current exchange.
:param pairs: list of pairs
:raise: OperationalException if one pair is not available
:return: None
"""
if not self.markets:
logger.warning("Unable to validate pairs (assuming they are correct).")
return
extended_pairs = expand_pairlist(pairs, list(self.markets), keep_invalid=True)
invalid_pairs = []
for pair in extended_pairs:
# Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
if self.markets and pair not in self.markets:
raise OperationalException(
f"Pair {pair} is not available on {self.name} {self.trading_mode}. "
f"Please remove {pair} from your whitelist."
)
# From ccxt Documentation:
# markets.info: An associative array of non-common market properties,
# including fees, rates, limits and other general market information.
# The internal info array is different for each particular market,
# its contents depend on the exchange.
# It can also be a string or similar ... so we need to verify that first.
elif isinstance(self.markets[pair].get("info"), dict) and self.markets[pair].get(
"info", {}
).get("prohibitedIn", False):
# Warn users about restricted pairs in whitelist.
# We cannot determine reliably if Users are affected.
logger.warning(
f"Pair {pair} is restricted for some users on this exchange."
f"Please check if you are impacted by this restriction "
f"on the exchange and eventually remove {pair} from your whitelist."
)
if (
self._config["stake_currency"]
and self.get_pair_quote_currency(pair) != self._config["stake_currency"]
):
invalid_pairs.append(pair)
if invalid_pairs:
raise OperationalException(
f"Stake-currency '{self._config['stake_currency']}' not compatible with "
f"pair-whitelist. Please remove the following pairs: {invalid_pairs}"
)
def get_valid_pair_combination(self, curr_1: str, curr_2: str) -> str:
"""
Get valid pair combination of curr_1 and curr_2 by trying both combinations.

View File

@ -61,14 +61,15 @@ class StaticPairList(IPairList):
:param tickers: Tickers (from exchange.get_tickers). May be cached.
:return: List of pairs
"""
if self._allow_inactive:
return self.verify_whitelist(
wl = self.verify_whitelist(
self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True
)
if self._allow_inactive:
return wl
else:
return self._whitelist_for_active_markets(
self.verify_whitelist(self._config["exchange"]["pair_whitelist"], logger.info)
)
# Avoid implicit filtering of "verify_whitelist" to keep
# proper warnings in the log
return self._whitelist_for_active_markets(wl)
def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]:
"""

View File

@ -28,6 +28,7 @@ def expand_pairlist(
except re.error as err:
raise ValueError(f"Wildcard error in {pair_wc}, {err}")
# Remove wildcard pairs that didn't have a match.
result = [element for element in result if re.fullmatch(r"^[A-Za-z0-9:/-]+$", element)]
else:

View File

@ -255,7 +255,6 @@ def test_init_exception(default_conf, mocker):
def test_exchange_resolver(default_conf, mocker, caplog):
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=MagicMock()))
mocker.patch(f"{EXMS}._load_async_markets")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
@ -555,7 +554,6 @@ def test_get_min_pair_stake_amount_real_data(mocker, default_conf) -> None:
def test__load_async_markets(default_conf, mocker, caplog):
mocker.patch(f"{EXMS}._init_ccxt")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.reload_markets")
mocker.patch(f"{EXMS}.validate_stakecurrency")
@ -584,7 +582,6 @@ def test__load_markets(default_conf, mocker, caplog):
api_mock = MagicMock()
api_mock.load_markets = get_mock_coro(side_effect=ccxt.BaseError("SomeError"))
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
@ -684,7 +681,6 @@ def test_validate_stakecurrency(default_conf, stake_currency, mocker, caplog):
}
)
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_pricing")
Exchange(default_conf)
@ -702,7 +698,6 @@ def test_validate_stakecurrency_error(default_conf, mocker, caplog):
}
)
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_timeframes")
with pytest.raises(
ConfigurationError,
@ -755,147 +750,6 @@ def test_get_pair_base_currency(default_conf, mocker, pair, expected):
assert ex.get_pair_base_currency(pair) == expected
def test_validate_pairs(default_conf, mocker):
api_mock = MagicMock()
id_mock = PropertyMock(return_value="test_exchange")
type(api_mock).id = id_mock
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(
f"{EXMS}._load_async_markets",
return_value={
"ETH/BTC": {"quote": "BTC"},
"LTC/BTC": {"quote": "BTC"},
"XRP/BTC": {"quote": "BTC"},
"NEO/BTC": {"quote": "BTC"},
},
)
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
# test exchange.validate_pairs directly
# No assert - but this should not fail (!)
Exchange(default_conf)
def test_validate_pairs_not_available(default_conf, mocker):
api_mock = MagicMock()
type(api_mock).markets = PropertyMock(
return_value={"XRP/BTC": {"inactive": True, "base": "XRP", "quote": "BTC"}}
)
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}._load_async_markets")
with pytest.raises(OperationalException, match=r"not available"):
Exchange(default_conf)
def test_validate_pairs_exception(default_conf, mocker, caplog):
caplog.set_level(logging.INFO)
api_mock = MagicMock()
mocker.patch(f"{EXMS}.name", PropertyMock(return_value="Binance"))
type(api_mock).markets = PropertyMock(return_value={})
mocker.patch(f"{EXMS}._init_ccxt", api_mock)
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
mocker.patch(f"{EXMS}._load_async_markets")
with pytest.raises(OperationalException, match=r"Pair ETH/BTC is not available on Binance"):
Exchange(default_conf)
mocker.patch(f"{EXMS}.markets", PropertyMock(return_value={}))
Exchange(default_conf)
assert log_has("Unable to validate pairs (assuming they are correct).", caplog)
def test_validate_pairs_restricted(default_conf, mocker, caplog):
api_mock = MagicMock()
type(api_mock).load_markets = get_mock_coro(
return_value={
"ETH/BTC": {"quote": "BTC"},
"LTC/BTC": {"quote": "BTC"},
"XRP/BTC": {"quote": "BTC", "info": {"prohibitedIn": ["US"]}},
"NEO/BTC": {"quote": "BTC", "info": "TestString"}, # info can also be a string ...
}
)
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_pricing")
mocker.patch(f"{EXMS}.validate_stakecurrency")
Exchange(default_conf)
assert log_has(
"Pair XRP/BTC is restricted for some users on this exchange."
"Please check if you are impacted by this restriction "
"on the exchange and eventually remove XRP/BTC from your whitelist.",
caplog,
)
def test_validate_pairs_stakecompatibility(default_conf, mocker):
api_mock = MagicMock()
type(api_mock).load_markets = get_mock_coro(
return_value={
"ETH/BTC": {"quote": "BTC"},
"LTC/BTC": {"quote": "BTC"},
"XRP/BTC": {"quote": "BTC"},
"NEO/BTC": {"quote": "BTC"},
"HELLO-WORLD": {"quote": "BTC"},
}
)
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
Exchange(default_conf)
def test_validate_pairs_stakecompatibility_downloaddata(default_conf, mocker):
api_mock = MagicMock()
default_conf["stake_currency"] = ""
type(api_mock).load_markets = get_mock_coro(
return_value={
"ETH/BTC": {"quote": "BTC"},
"LTC/BTC": {"quote": "BTC"},
"XRP/BTC": {"quote": "BTC"},
"NEO/BTC": {"quote": "BTC"},
"HELLO-WORLD": {"quote": "BTC"},
}
)
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
Exchange(default_conf)
assert type(api_mock).load_markets.call_count == 1
def test_validate_pairs_stakecompatibility_fail(default_conf, mocker):
default_conf["exchange"]["pair_whitelist"].append("HELLO-WORLD")
api_mock = MagicMock()
type(api_mock).load_markets = get_mock_coro(
return_value={
"ETH/BTC": {"quote": "BTC"},
"LTC/BTC": {"quote": "BTC"},
"XRP/BTC": {"quote": "BTC"},
"NEO/BTC": {"quote": "BTC"},
"HELLO-WORLD": {"quote": "USDT"},
}
)
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_stakecurrency")
with pytest.raises(OperationalException, match=r"Stake-currency 'BTC' not compatible with.*"):
Exchange(default_conf)
@pytest.mark.parametrize("timeframe", [("5m"), ("1m"), ("15m"), ("1h")])
def test_validate_timeframes(default_conf, mocker, timeframe):
default_conf["timeframe"] = timeframe
@ -907,7 +761,6 @@ def test_validate_timeframes(default_conf, mocker, timeframe):
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.reload_markets")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
Exchange(default_conf)
@ -925,7 +778,6 @@ def test_validate_timeframes_failed(default_conf, mocker):
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.reload_markets")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
with pytest.raises(
@ -955,7 +807,6 @@ def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker):
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.reload_markets")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_stakecurrency")
with pytest.raises(
OperationalException,
@ -977,7 +828,6 @@ def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker):
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.reload_markets")
mocker.patch(f"{EXMS}.validate_pairs", MagicMock())
mocker.patch(f"{EXMS}.validate_stakecurrency")
with pytest.raises(
OperationalException,
@ -999,7 +849,6 @@ def test_validate_timeframes_not_in_config(default_conf, mocker):
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.reload_markets")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
mocker.patch(f"{EXMS}.validate_required_startup_candles")
@ -1016,7 +865,6 @@ def test_validate_pricing(default_conf, mocker):
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.reload_markets")
mocker.patch(f"{EXMS}.validate_trading_mode_and_margin_mode")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.name", "Binance")
@ -1051,7 +899,6 @@ def test_validate_ordertypes(default_conf, mocker):
type(api_mock).has = PropertyMock(return_value={"createMarketOrder": True})
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.reload_markets")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
@ -1110,7 +957,6 @@ def test_validate_ordertypes_stop_advanced(default_conf, mocker, exchange_name,
type(api_mock).has = PropertyMock(return_value={"createMarketOrder": True})
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.reload_markets")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_stakecurrency")
mocker.patch(f"{EXMS}.validate_pricing")
@ -1135,7 +981,6 @@ def test_validate_order_types_not_in_config(default_conf, mocker):
api_mock = MagicMock()
mocker.patch(f"{EXMS}._init_ccxt", MagicMock(return_value=api_mock))
mocker.patch(f"{EXMS}.reload_markets")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}.validate_pricing")
mocker.patch(f"{EXMS}.validate_stakecurrency")
@ -1151,7 +996,6 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog):
mocker.patch(f"{EXMS}._init_ccxt", api_mock)
mocker.patch(f"{EXMS}.validate_timeframes")
mocker.patch(f"{EXMS}._load_async_markets")
mocker.patch(f"{EXMS}.validate_pairs")
mocker.patch(f"{EXMS}.validate_pricing")
mocker.patch(f"{EXMS}.validate_stakecurrency")
@ -4185,7 +4029,6 @@ def test_merge_ft_has_dict(default_conf, mocker):
EXMS,
_init_ccxt=MagicMock(return_value=MagicMock()),
_load_async_markets=MagicMock(),
validate_pairs=MagicMock(),
validate_timeframes=MagicMock(),
validate_stakecurrency=MagicMock(),
validate_pricing=MagicMock(),
@ -4220,7 +4063,6 @@ def test_get_valid_pair_combination(default_conf, mocker, markets):
EXMS,
_init_ccxt=MagicMock(return_value=MagicMock()),
_load_async_markets=MagicMock(),
validate_pairs=MagicMock(),
validate_timeframes=MagicMock(),
validate_pricing=MagicMock(),
markets=PropertyMock(return_value=markets),
@ -4500,7 +4342,6 @@ def test_get_markets(
EXMS,
_init_ccxt=MagicMock(return_value=MagicMock()),
_load_async_markets=MagicMock(),
validate_pairs=MagicMock(),
validate_timeframes=MagicMock(),
validate_pricing=MagicMock(),
markets=PropertyMock(return_value=markets_static),

View File

@ -2204,7 +2204,6 @@ def test_manage_open_orders_buy_exception(
patch_exchange(mocker)
mocker.patch.multiple(
EXMS,
validate_pairs=MagicMock(),
fetch_ticker=ticker_usdt,
fetch_order=MagicMock(side_effect=ExchangeError),
cancel_order=cancel_order_mock,