Merge branch 'develop' into ci/ccxt.pro

.github/workflows/ci.yml

@@ -55,7 +55,7 @@ jobs:
 
     - name: Installation - *nix
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -192,7 +192,7 @@ jobs:
 
     - name: Installation (python)
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -422,7 +422,7 @@ jobs:
 
     - name: Installation - *nix
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include

.pre-commit-config.yaml

@@ -9,17 +9,17 @@ repos:
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.0"
+    rev: "v1.10.1"
     hooks:
       - id: mypy
         exclude: build_helpers
         additional_dependencies:
           - types-cachetools==5.3.0.7
           - types-filelock==3.2.7
-          - types-requests==2.32.0.20240602
+          - types-requests==2.32.0.20240622
           - types-tabulate==0.9.0.20240106
           - types-python-dateutil==2.9.0.20240316
-          - SQLAlchemy==2.0.30
+          - SQLAlchemy==2.0.31
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.4.9'
+    rev: 'v0.5.0'
     hooks:
       - id: ruff
 

Dockerfile

@@ -25,7 +25,7 @@ FROM base as python-deps
 RUN  apt-get update \
   && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \
   && apt-get clean \
-  && pip install --upgrade pip wheel
+  && pip install --upgrade "pip<=24.0" wheel
 
 # Install TA-lib
 COPY build_helpers/* /tmp/
@@ -35,7 +35,7 @@ ENV LD_LIBRARY_PATH /usr/local/lib
 # Install dependencies
 COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/
 USER ftuser
-RUN  pip install --user --no-cache-dir numpy \
+RUN  pip install --user --no-cache-dir "numpy<2.0" \
   && pip install --user --no-cache-dir -r requirements-hyperopt.txt
 
 # Copy dependencies to runtime-image

build_helpers/install_windows.ps1

@@ -1,6 +1,6 @@
 # vendored Wheels compiled via https://github.com/xmatthias/ta-lib-python/tree/ta_bundled_040
 
-python -m pip install --upgrade pip wheel
+python -m pip install --upgrade "pip<=24.0" wheel
 
 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
 

docker/Dockerfile.armhf

@@ -17,7 +17,7 @@ RUN mkdir /freqtrade \
   && chown ftuser:ftuser /freqtrade \
   # Allow sudoers
   && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers \
-  && pip install --upgrade pip
+  && pip install --upgrade "pip<=24.0"
 
 WORKDIR /freqtrade
 

docs/configuration.md

@@ -650,9 +650,9 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo
 * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode.
 * Wallets (`/balance`) are simulated based on `dry_run_wallet`.
 * Orders are simulated, and will not be posted to the exchange.
-* Market orders fill based on orderbook volume the moment the order is placed.
+* Market orders fill based on orderbook volume the moment the order is placed, with a maximum slippage of 5%.
 * Limit orders fill once the price reaches the defined level - or time out based on `unfilledtimeout` settings.
-* Limit orders will be converted to market orders if they cross the price by more than 1%.
+* Limit orders will be converted to market orders if they cross the price by more than 1%, and will be filled immediately based regular market order rules (see point about Market orders above).
 * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled.
 * Open orders (not trades, which are stored in the database) are kept open after bot restarts, with the assumption that they were not filled while being offline.
 

docs/includes/pairlists.md

@@ -373,7 +373,7 @@ Filters low-value coins which would not allow setting stoplosses.
 
 Namely, pairs are blacklisted if a variance of one percent or more in the stop price would be caused by precision rounding on the exchange, i.e. `rounded(stop_price) <= rounded(stop_price * 0.99)`. The idea is to avoid coins with a value VERY close to their lower trading boundary, not allowing setting of proper stoploss.
 
-!!! Tip "PerformanceFilter is pointless for futures trading"
+!!! Tip "PrecisionFilter is pointless for futures trading"
     The above does not apply to shorts. And for longs, in theory the trade will be liquidated first.
 
 !!! Warning "Backtesting"

docs/plotting.md

@@ -2,6 +2,14 @@
 
 This page explains how to plot prices, indicators and profits.
 
+!!! Warning "Deprecated"
+    The commands described in this page (`plot-dataframe`, `plot-profit`) should be considered deprecated and are in maintenance mode.
+    This is mostly for the performance problems even medium sized plots can cause, but also because "store a file and open it in a browser" isn't very intuitive from a UI perspective.
+
+    While there are no immediate plans to remove them, they are not actively maintained - and may be removed short-term should major changes be required to keep them working.
+    
+    Please use [FreqUI](freq-ui.md) for plotting needs, which doesn't struggle with the same performance problems.
+
 ## Installation / Setup
 
 Plotting modules use the Plotly library. You can install / upgrade this by running the following command:

docs/strategy-callbacks.md

@@ -165,7 +165,9 @@ E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoplo
 During backtesting, `current_rate` (and `current_profit`) are provided against the candle's high (or low for short trades) - while the resulting stoploss is evaluated against the candle's low (or high for short trades).
 
 The absolute value of the return value is used (the sign is ignored), so returning `0.05` or `-0.05` have the same result, a stoploss 5% below the current price.
-Returning None will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+Returning `None` will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+`NaN` and `inf` values are considered invalid and will be ignored (identical to `None`).
+
 
 Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchangefreqtrade)).
 

freqtrade/__init__.py

@@ -1,6 +1,6 @@
 """Freqtrade bot"""
 
-__version__ = "2024.6-dev"
+__version__ = "2024.7-dev"
 
 if "dev" in __version__:
     from pathlib import Path

freqtrade/commands/arguments.py

@@ -2,10 +2,10 @@
 This module contains the argument manager class
 """
 
-import argparse
+from argparse import ArgumentParser, Namespace, _ArgumentGroup
 from functools import partial
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS
 from freqtrade.constants import DEFAULT_CONFIG
@@ -226,6 +226,19 @@ ARGS_ANALYZE_ENTRIES_EXITS = [
     "analysis_csv_path",
 ]
 
+
+ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
+
+ARGS_LOOKAHEAD_ANALYSIS = [
+    a
+    for a in ARGS_BACKTEST
+    if a
+    not in ("position_stacking", "use_max_market_positions", "backtest_cache", "backtest_breakdown")
+] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
+
+ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
+
+# Command level configs - keep at the bottom of the above definitions
 NO_CONF_REQURIED = [
     "convert-data",
     "convert-trade-data",
@@ -248,14 +261,6 @@ NO_CONF_REQURIED = [
 
 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
 
-ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
-
-ARGS_LOOKAHEAD_ANALYSIS = [
-    a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", "cache")
-] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
-
-ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
-
 
 class Arguments:
     """
@@ -264,7 +269,7 @@ class Arguments:
 
     def __init__(self, args: Optional[List[str]]) -> None:
         self.args = args
-        self._parsed_arg: Optional[argparse.Namespace] = None
+        self._parsed_arg: Optional[Namespace] = None
 
     def get_parsed_arg(self) -> Dict[str, Any]:
         """
@@ -277,7 +282,7 @@ class Arguments:
 
         return vars(self._parsed_arg)
 
-    def _parse_args(self) -> argparse.Namespace:
+    def _parse_args(self) -> Namespace:
         """
         Parses given arguments and returns an argparse Namespace instance.
         """
@@ -306,7 +311,9 @@ class Arguments:
 
         return parsed_arg
 
-    def _build_args(self, optionlist, parser):
+    def _build_args(
+        self, optionlist: List[str], parser: Union[ArgumentParser, _ArgumentGroup]
+    ) -> None:
         for val in optionlist:
             opt = AVAILABLE_CLI_OPTIONS[val]
             parser.add_argument(*opt.cli, dest=val, **opt.kwargs)
@@ -317,16 +324,16 @@ class Arguments:
         :return: None
         """
         # Build shared arguments (as group Common Options)
-        _common_parser = argparse.ArgumentParser(add_help=False)
+        _common_parser = ArgumentParser(add_help=False)
         group = _common_parser.add_argument_group("Common arguments")
         self._build_args(optionlist=ARGS_COMMON, parser=group)
 
-        _strategy_parser = argparse.ArgumentParser(add_help=False)
+        _strategy_parser = ArgumentParser(add_help=False)
         strategy_group = _strategy_parser.add_argument_group("Strategy arguments")
         self._build_args(optionlist=ARGS_STRATEGY, parser=strategy_group)
 
         # Build main command
-        self.parser = argparse.ArgumentParser(
+        self.parser = ArgumentParser(
             prog="freqtrade", description="Free, open source crypto trading bot"
         )
         self._build_args(optionlist=["version"], parser=self.parser)

freqtrade/data/__init__.py

@@ -2,5 +2,8 @@
 Module to handle data operations for freqtrade
 """
 
+from freqtrade.data import converter
+
+
 # limit what's imported when using `from freqtrade.data import *`
 __all__ = ["converter"]

freqtrade/data/history/history_utils.py

@@ -618,6 +618,11 @@ def download_data_main(config: Config) -> None:
     # Start downloading
     try:
         if config.get("download_trades"):
+            if not exchange.get_option("trades_has_history", True):
+                raise OperationalException(
+                    f"Trade history not available for {exchange.name}. "
+                    "You cannot use --dl-trades for this exchange."
+                )
             pairs_not_available = refresh_backtest_trades_data(
                 exchange,
                 pairs=expanded_pairs,

freqtrade/exchange/binance.py

@@ -28,6 +28,7 @@ class Binance(Exchange):
         "ohlcv_candle_limit": 1000,
         "trades_pagination": "id",
         "trades_pagination_arg": "fromId",
+        "trades_has_history": True,
         "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000],
         "ws.enabled": True,
     }

freqtrade/exchange/bingx.py

@@ -20,4 +20,5 @@ class Bingx(Exchange):
         "stoploss_on_exchange": True,
         "stoploss_order_types": {"limit": "limit", "market": "market"},
         "order_time_in_force": ["GTC", "IOC", "PO"],
+        "trades_has_history": False,  # Endpoint doesn't seem to support pagination
     }

freqtrade/exchange/bitmart.py

@@ -18,4 +18,5 @@ class Bitmart(Exchange):
     _ft_has: Dict = {
         "stoploss_on_exchange": False,  # Bitmart API does not support stoploss orders
         "ohlcv_candle_limit": 200,
+        "trades_has_history": False,  # Endpoint doesn't seem to support pagination
     }

freqtrade/exchange/bybit.py

@@ -34,6 +34,7 @@ class Bybit(Exchange):
         "ohlcv_has_history": True,
         "order_time_in_force": ["GTC", "FOK", "IOC", "PO"],
         "ws.enabled": True,
+        "trades_has_history": False,  # Endpoint doesn't support pagination
     }
     _ft_has_futures: Dict = {
         "ohlcv_has_history": True,

freqtrade/exchange/exchange.py

@@ -125,6 +125,7 @@ class Exchange:
         "tickers_have_price": True,
         "trades_pagination": "time",  # Possible are "time" or "id"
         "trades_pagination_arg": "since",
+        "trades_has_history": False,
         "l2_limit_range": None,
         "l2_limit_range_required": True,  # Allow Empty L2 limit (kucoin)
         "mark_ohlcv_price": "mark",

freqtrade/exchange/gate.py

@@ -31,6 +31,7 @@ class Gate(Exchange):
         "stop_price_param": "stopPrice",
         "stop_price_prop": "stopPrice",
         "marketOrderRequiresPrice": True,
+        "trades_has_history": False,  # Endpoint would support this - but ccxt doesn't.
     }
 
     _ft_has_futures: Dict = {

freqtrade/exchange/htx.py

@@ -28,6 +28,7 @@ class Htx(Exchange):
             "1w": 500,
             "1M": 500,
         },
+        "trades_has_history": False,  # Endpoint doesn't have a "since" parameter
     }
 
     def _get_stop_params(self, side: BuySell, ordertype: str, stop_price: float) -> Dict:

freqtrade/exchange/kraken.py

@@ -31,6 +31,7 @@ class Kraken(Exchange):
         "trades_pagination": "id",
         "trades_pagination_arg": "since",
         "trades_pagination_overlap": False,
+        "trades_has_history": True,
         "mark_ohlcv_timeframe": "4h",
     }
 

freqtrade/exchange/okx.py

@@ -33,6 +33,7 @@ class Okx(Exchange):
         "funding_fee_timeframe": "8h",
         "stoploss_order_types": {"limit": "limit"},
         "stoploss_on_exchange": True,
+        "trades_has_history": False,  # Endpoint doesn't have a "since" parameter
     }
     _ft_has_futures: Dict = {
         "tickers_have_quoteVolume": False,

freqtrade/misc.py

@@ -33,7 +33,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
         if log:
             logger.info(f'dumping json to "{filename}"')
 
-        with gzip.open(filename, "w") as fpz:
+        with gzip.open(filename, "wt", encoding="utf-8") as fpz:
             rapidjson.dump(data, fpz, default=str, number_mode=rapidjson.NM_NATIVE)
     else:
         if log:
@@ -60,7 +60,7 @@ def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None:
     logger.debug(f'done joblib dump to "{filename}"')
 
 
-def json_load(datafile: Union[gzip.GzipFile, TextIO]) -> Any:
+def json_load(datafile: TextIO) -> Any:
     """
     load data with rapidjson
     Use this to have a consistent experience,
@@ -77,7 +77,7 @@ def file_load_json(file: Path):
     # Try gzip file first, otherwise regular json file.
     if gzipfile.is_file():
         logger.debug(f"Loading historical data from file {gzipfile}")
-        with gzip.open(gzipfile) as datafile:
+        with gzip.open(gzipfile, "rt", encoding="utf-8") as datafile:
             pairdata = json_load(datafile)
     elif file.is_file():
         logger.debug(f"Loading historical data from file {file}")

freqtrade/optimize/backtesting.py

@@ -217,8 +217,6 @@ class Backtesting:
             raise OperationalException(
                 "VolumePairList not allowed for backtesting. Please use StaticPairList instead."
             )
-        if "PerformanceFilter" in self.pairlists.name_list:
-            raise OperationalException("PerformanceFilter not allowed for backtesting.")
 
         if len(self.strategylist) > 1 and "PrecisionFilter" in self.pairlists.name_list:
             raise OperationalException(

freqtrade/plugins/pairlist/AgeFilter.py

@@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import PeriodicCache, dt_floor_day, dt_now, dt_ts
 
 
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
 
 
 class AgeFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/FullTradesFilter.py

@@ -7,13 +7,15 @@ from typing import List
 
 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class FullTradesFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO_ACTION
+
     @property
     def needstickers(self) -> bool:
         """

freqtrade/plugins/pairlist/IPairList.py

@@ -5,6 +5,7 @@ PairList Handler base class
 import logging
 from abc import ABC, abstractmethod
 from copy import deepcopy
+from enum import Enum
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
 
 from freqtrade.constants import Config
@@ -51,8 +52,20 @@ PairlistParameter = Union[
 ]
 
 
+class SupportsBacktesting(str, Enum):
+    """
+    Enum to indicate if a Pairlist Handler supports backtesting.
+    """
+
+    YES = "yes"
+    NO = "no"
+    NO_ACTION = "no_action"
+    BIASED = "biased"
+
+
 class IPairList(LoggingMixin, ABC):
     is_pairlist_generator = False
+    supports_backtesting: SupportsBacktesting = SupportsBacktesting.NO
 
     def __init__(
         self,

freqtrade/plugins/pairlist/MarketCapPairList.py

@@ -11,7 +11,7 @@ from cachetools import TTLCache
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.coin_gecko import FtCoinGeckoApi
 
 
@@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
 
 class MarketCapPairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.BIASED
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/OffsetFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, List
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class OffsetFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.YES
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PerformanceFilter.py

@@ -9,13 +9,15 @@ import pandas as pd
 
 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PerformanceFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO_ACTION
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PrecisionFilter.py

@@ -8,13 +8,15 @@ from typing import Optional
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import ROUND_UP
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PrecisionFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.BIASED
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PriceFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PriceFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.BIASED
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/ProducerPairList.py

@@ -9,7 +9,7 @@ from typing import Dict, List, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
@@ -31,6 +31,7 @@ class ProducerPairList(IPairList):
     """
 
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.NO
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/RemotePairList.py

@@ -16,7 +16,7 @@ from freqtrade import __version__
 from freqtrade.configuration.load_config import CONFIG_PARSE_MODE
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 
 
@@ -25,6 +25,8 @@ logger = logging.getLogger(__name__)
 
 class RemotePairList(IPairList):
     is_pairlist_generator = True
+    # Potential winner bias
+    supports_backtesting = SupportsBacktesting.BIASED
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/ShuffleFilter.py

@@ -9,7 +9,7 @@ from typing import Dict, List, Literal
 from freqtrade.enums import RunMode
 from freqtrade.exchange import timeframe_to_seconds
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.periodic_cache import PeriodicCache
 
 
@@ -19,6 +19,8 @@ ShuffleValues = Literal["candle", "iteration"]
 
 
 class ShuffleFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.YES
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/SpreadFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class SpreadFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/StaticPairList.py

@@ -9,7 +9,7 @@ from copy import deepcopy
 from typing import Dict, List
 
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
@@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
 
 class StaticPairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.YES
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -15,7 +15,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_floor_day, dt_now, dt_ts
 
 
@@ -27,6 +27,8 @@ class VolatilityFilter(IPairList):
     Filters pairs by volatility
     """
 
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/VolumePairList.py

@@ -14,7 +14,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_now, format_ms_time
 
 
@@ -26,6 +26,7 @@ SORT_VALUES = ["quoteVolume"]
 
 class VolumePairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.NO
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_floor_day, dt_now, dt_ts
 
 
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
 
 
 class RangeStabilityFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlistmanager.py

@@ -11,10 +11,11 @@ from cachetools import TTLCache, cached
 from freqtrade.constants import Config, ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import CandleType
+from freqtrade.enums.runmode import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.mixins import LoggingMixin
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import PairListResolver
 
@@ -57,9 +58,44 @@ class PairListManager(LoggingMixin):
                 f"{invalid}."
             )
 
+        self._check_backtest()
+
         refresh_period = config.get("pairlist_refresh_period", 3600)
         LoggingMixin.__init__(self, logger, refresh_period)
 
+    def _check_backtest(self) -> None:
+        if self._config["runmode"] not in (RunMode.BACKTEST, RunMode.EDGE, RunMode.HYPEROPT):
+            return
+
+        pairlist_errors: List[str] = []
+        noaction_pairlists: List[str] = []
+        biased_pairlists: List[str] = []
+        for pairlist_handler in self._pairlist_handlers:
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.NO:
+                pairlist_errors.append(pairlist_handler.name)
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.NO_ACTION:
+                noaction_pairlists.append(pairlist_handler.name)
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.BIASED:
+                biased_pairlists.append(pairlist_handler.name)
+
+        if noaction_pairlists:
+            logger.warning(
+                f"Pairlist Handlers {', '.join(noaction_pairlists)} do not generate "
+                "any changes during backtesting. While it's safe to leave them enabled, they will "
+                "not behave like in dry/live modes. "
+            )
+
+        if biased_pairlists:
+            logger.warning(
+                f"Pairlist Handlers {', '.join(biased_pairlists)} will introduce a lookahead bias "
+                "to your backtest results, as they use today's data - which inheritly suffers from "
+                "'winner bias'."
+            )
+        if pairlist_errors:
+            raise OperationalException(
+                f"Pairlist Handlers {', '.join(pairlist_errors)} do not support backtesting."
+            )
+
     @property
     def whitelist(self) -> List[str]:
         """The current whitelist"""

freqtrade/rpc/telegram.py

@@ -1787,7 +1787,7 @@ class Telegram(RPCHandler):
             "_Bot Control_\n"
             "------------\n"
             "*/start:* `Starts the trader`\n"
-            "*/stop:* Stops the trader\n"
+            "*/stop:* `Stops the trader`\n"
             "*/stopentry:* `Stops entering, but handles open trades gracefully` \n"
             "*/forceexit <trade_id>|all:* `Instantly exits the given trade or all trades, "
             "regardless of profit`\n"
@@ -1820,7 +1820,7 @@ class Telegram(RPCHandler):
             "that represents the current market direction. If no direction is provided `"
             "`the currently set market direction will be output.` \n"
             "*/list_custom_data <trade_id> <key>:* `List custom_data for Trade ID & Key combo.`\n"
-            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`"
+            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`\n"
             "_Statistics_\n"
             "------------\n"
             "*/status <trade_id>|[table]:* `Lists all open trades`\n"

freqtrade/strategy/interface.py

@@ -6,6 +6,7 @@ This module defines the interface to apply for strategies
 import logging
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta, timezone
+from math import isinf, isnan
 from typing import Dict, List, Optional, Tuple, Union
 
 from pandas import DataFrame
@@ -1423,7 +1424,9 @@ class IStrategy(ABC, HyperStrategyMixin):
                 after_fill=after_fill,
             )
             # Sanity check - error cases will return None
-            if stop_loss_value_custom:
+            if stop_loss_value_custom and not (
+                isnan(stop_loss_value_custom) or isinf(stop_loss_value_custom)
+            ):
                 stop_loss_value = stop_loss_value_custom
                 trade.adjust_stop_loss(
                     bound or current_rate, stop_loss_value, allow_refresh=after_fill

ft_client/freqtrade_client/__init__.py

@@ -1,7 +1,7 @@
 from freqtrade_client.ft_rest_client import FtRestClient
 
 
-__version__ = "2024.6-dev"
+__version__ = "2024.7-dev"
 
 if "dev" in __version__:
     from pathlib import Path

ft_client/requirements.txt

@@ -1,3 +1,3 @@
 # Requirements for freqtrade client library
 requests==2.32.3
-python-rapidjson==1.17
+python-rapidjson==1.18

pyproject.toml

@@ -144,6 +144,7 @@ extend-select = [
   # "TCH",  # flake8-type-checking
   "PTH",    # flake8-use-pathlib
   # "RUF",    # ruff
+  "ASYNC",  # flake8-async
 ]
 
 extend-ignore = [

requirements-dev.txt

@@ -7,8 +7,8 @@
 -r docs/requirements-docs.txt
 
 coveralls==4.0.1
-ruff==0.4.9
-mypy==1.10.0
+ruff==0.5.0
+mypy==1.10.1
 pre-commit==3.7.1
 pytest==8.2.2
 pytest-asyncio==0.23.7
@@ -19,7 +19,7 @@ pytest-timeout==2.3.1
 pytest-xdist==3.6.1
 isort==5.13.2
 # For datetime mocking
-time-machine==2.14.1
+time-machine==2.14.2
 
 # Convert jupyter notebooks to markdown documents
 nbconvert==7.16.4
@@ -27,6 +27,6 @@ nbconvert==7.16.4
 # mypy types
 types-cachetools==5.3.0.7
 types-filelock==3.2.7
-types-requests==2.32.0.20240602
+types-requests==2.32.0.20240622
 types-tabulate==0.9.0.20240106
 types-python-dateutil==2.9.0.20240316

requirements-hyperopt.txt

@@ -2,7 +2,8 @@
 -r requirements.txt
 
 # Required for hyperopt
-scipy==1.13.1
+scipy==1.14.0; python_version >= "3.10"
+scipy==1.13.1; python_version < "3.10"
 scikit-learn==1.5.0
 ft-scikit-optimize==0.9.2
-filelock==3.15.1
+filelock==3.15.4

requirements.txt

@@ -1,13 +1,13 @@
 numpy==1.26.4
 pandas==2.2.2
-bottleneck==1.3.8
-numexpr==2.10.0
+bottleneck==1.4.0
+numexpr==2.10.1
 pandas-ta==0.3.14b
 
-ccxt==4.3.46
+ccxt==4.3.54
 cryptography==42.0.8
 aiohttp==3.9.5
-SQLAlchemy==2.0.30
+SQLAlchemy==2.0.31
 python-telegram-bot==21.3
 # can't be hard-pinned due to telegram-bot pinning httpx with ~
 httpx>=0.24.1
@@ -16,7 +16,7 @@ cachetools==5.3.3
 requests==2.32.3
 urllib3==2.2.2
 jsonschema==4.22.0
-TA-Lib==0.4.31
+TA-Lib==0.4.32
 technical==1.4.3
 tabulate==0.9.0
 pycoingecko==3.1.0
@@ -30,7 +30,7 @@ pyarrow==16.1.0; platform_machine != 'armv7l'
 py_find_1st==1.1.6
 
 # Load ticker files 30% faster
-python-rapidjson==1.17
+python-rapidjson==1.18
 # Properly format api responses
 orjson==3.10.5
 
@@ -42,8 +42,8 @@ fastapi==0.111.0
 pydantic==2.7.4
 uvicorn==0.30.1
 pyjwt==2.8.0
-aiofiles==23.2.1
-psutil==5.9.8
+aiofiles==24.1.0
+psutil==6.0.0
 
 # Support for colorized terminal output
 colorama==0.4.6

setup.sh

@@ -49,7 +49,7 @@ function updateenv() {
     source .venv/bin/activate
     SYS_ARCH=$(uname -m)
     echo "pip install in-progress. Please wait..."
-    ${PYTHON} -m pip install --upgrade pip wheel setuptools
+    ${PYTHON} -m pip install --upgrade "pip<=24.0" wheel setuptools
     REQUIREMENTS_HYPEROPT=""
     REQUIREMENTS_PLOT=""
     REQUIREMENTS_FREQAI=""

tests/data/test_download_data.py

@@ -83,6 +83,12 @@ def test_download_data_main_trades(mocker):
     assert dl_mock.call_count == 1
     assert convert_mock.call_count == 1
 
+    # Exchange that doesn't support historic downloads
+    config["exchange"]["name"] = "bybit"
+    with pytest.raises(OperationalException, match=r"Trade history not available for .*"):
+        config
+        download_data_main(config)
+
 
 def test_download_data_main_data_invalid(mocker):
     patch_exchange(mocker, id="kraken")

tests/optimize/test_backtesting.py

@@ -429,7 +429,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
         backtesting.start()
 
 
-def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> None:
+def test_backtesting_no_pair_left(default_conf, mocker) -> None:
     mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True))
     mocker.patch(
         "freqtrade.data.history.history_utils.load_pair_history",
@@ -449,13 +449,6 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) ->
     with pytest.raises(OperationalException, match="No pair in whitelist."):
         Backtesting(default_conf)
 
-    default_conf["pairlists"] = [{"method": "VolumePairList", "number_assets": 5}]
-    with pytest.raises(
-        OperationalException,
-        match=r"VolumePairList not allowed for backtesting\..*StaticPairList.*",
-    ):
-        Backtesting(default_conf)
-
     default_conf.update(
         {
             "pairlists": [{"method": "StaticPairList"}],
@@ -469,7 +462,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) ->
         Backtesting(default_conf)
 
 
-def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, tickers) -> None:
+def test_backtesting_pairlist_list(default_conf, mocker, tickers) -> None:
     mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True))
     mocker.patch(f"{EXMS}.get_tickers", tickers)
     mocker.patch(f"{EXMS}.price_to_precision", lambda s, x, y: y)
@@ -495,12 +488,6 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti
     ):
         Backtesting(default_conf)
 
-    default_conf["pairlists"] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}]
-    with pytest.raises(
-        OperationalException, match="PerformanceFilter not allowed for backtesting."
-    ):
-        Backtesting(default_conf)
-
     default_conf["pairlists"] = [
         {"method": "StaticPairList"},
         {"method": "PrecisionFilter"},

tests/plugins/test_pairlist.py

@@ -38,6 +38,7 @@ TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairLis
 
 @pytest.fixture(scope="function")
 def whitelist_conf(default_conf):
+    default_conf["runmode"] = "dry_run"
     default_conf["stake_currency"] = "BTC"
     default_conf["exchange"]["pair_whitelist"] = [
         "ETH/BTC",
@@ -68,6 +69,7 @@ def whitelist_conf(default_conf):
 
 @pytest.fixture(scope="function")
 def whitelist_conf_2(default_conf):
+    default_conf["runmode"] = "dry_run"
     default_conf["stake_currency"] = "BTC"
     default_conf["exchange"]["pair_whitelist"] = [
         "ETH/BTC",
@@ -94,6 +96,7 @@ def whitelist_conf_2(default_conf):
 
 @pytest.fixture(scope="function")
 def whitelist_conf_agefilter(default_conf):
+    default_conf["runmode"] = "dry_run"
     default_conf["stake_currency"] = "BTC"
     default_conf["exchange"]["pair_whitelist"] = [
         "ETH/BTC",
@@ -773,7 +776,7 @@ def test_VolumePairList_whitelist_gen(
     whitelist_result,
     caplog,
 ) -> None:
-    whitelist_conf["runmode"] = "backtest"
+    whitelist_conf["runmode"] = "util_exchange"
     whitelist_conf["pairlists"] = pairlists
     whitelist_conf["stake_currency"] = base_currency
 
@@ -2387,3 +2390,65 @@ def test_MarketCapPairList_exceptions(mocker, default_conf_usdt):
         OperationalException, match="This filter only support marketcap rank up to 250."
     ):
         PairListManager(exchange, default_conf_usdt)
+
+
+@pytest.mark.parametrize(
+    "pairlists,expected_error,expected_warning",
+    [
+        (
+            [{"method": "StaticPairList"}],
+            None,  # Error
+            None,  # Warning
+        ),
+        (
+            [{"method": "VolumePairList", "number_assets": 10}],
+            "VolumePairList",  # Error
+            None,  # Warning
+        ),
+        (
+            [{"method": "MarketCapPairList", "number_assets": 10}],
+            None,  # Error
+            r"MarketCapPairList.*lookahead.*",  # Warning
+        ),
+        (
+            [{"method": "StaticPairList"}, {"method": "FullTradesFilter"}],
+            None,  # Error
+            r"FullTradesFilter do not generate.*",  # Warning
+        ),
+        (  # combi, fails and warns
+            [
+                {"method": "VolumePairList", "number_assets": 10},
+                {"method": "MarketCapPairList", "number_assets": 10},
+            ],
+            "VolumePairList",  # Error
+            r"MarketCapPairList.*lookahead.*",  # Warning
+        ),
+    ],
+)
+def test_backtesting_modes(
+    mocker, default_conf_usdt, pairlists, expected_error, expected_warning, caplog, markets, tickers
+):
+    default_conf_usdt["runmode"] = "dry_run"
+    default_conf_usdt["pairlists"] = pairlists
+
+    mocker.patch.multiple(
+        EXMS,
+        markets=PropertyMock(return_value=markets),
+        exchange_has=MagicMock(return_value=True),
+        get_tickers=tickers,
+    )
+    exchange = get_patched_exchange(mocker, default_conf_usdt)
+
+    # Dry run mode - works always
+    PairListManager(exchange, default_conf_usdt)
+
+    default_conf_usdt["runmode"] = "backtest"
+    if expected_error:
+        with pytest.raises(OperationalException, match=f"Pairlist Handlers {expected_error}.*"):
+            PairListManager(exchange, default_conf_usdt)
+
+    if not expected_error:
+        PairListManager(exchange, default_conf_usdt)
+
+    if expected_warning:
+        assert log_has_re(f"Pairlist Handlers {expected_warning}", caplog)

tests/strategy/test_interface.py

@@ -1,5 +1,6 @@
 # pragma pylint: disable=missing-docstring, C0103
 import logging
+import math
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from unittest.mock import MagicMock
@@ -458,55 +459,66 @@ def test_min_roi_reached3(default_conf, fee) -> None:
             ExitType.TRAILING_STOP_LOSS,
             None,
         ),
-        (0.01, 0.96, ExitType.NONE, None, True, False, 0.05, 1, ExitType.NONE, None),
-        (0.05, 1, ExitType.NONE, None, True, False, -0.01, 1, ExitType.TRAILING_STOP_LOSS, None),
+        (0.01, 0.96, ExitType.NONE, None, True, False, 0.05, 0.998, ExitType.NONE, None),
+        (
+            0.05,
+            0.998,
+            ExitType.NONE,
+            None,
+            True,
+            False,
+            -0.01,
+            0.998,
+            ExitType.TRAILING_STOP_LOSS,
+            None,
+        ),
         # Default custom case - trails with 10%
-        (0.05, 0.95, ExitType.NONE, None, False, True, -0.02, 0.95, ExitType.NONE, None),
+        (0.05, 0.945, ExitType.NONE, None, False, True, -0.02, 0.945, ExitType.NONE, None),
         (
             0.05,
-            0.95,
+            0.945,
             ExitType.NONE,
             None,
             False,
             True,
             -0.06,
-            0.95,
+            0.945,
             ExitType.TRAILING_STOP_LOSS,
             None,
         ),
         (
             0.05,
-            1,
+            0.998,
             ExitType.NONE,
             None,
             False,
             True,
             -0.06,
-            1,
+            0.998,
             ExitType.TRAILING_STOP_LOSS,
             lambda **kwargs: -0.05,
         ),
         (
             0.05,
-            1,
+            0.998,
             ExitType.NONE,
             None,
             False,
             True,
             0.09,
-            1.04,
+            1.036,
             ExitType.NONE,
             lambda **kwargs: -0.05,
         ),
         (
             0.05,
-            0.95,
+            0.945,
             ExitType.NONE,
             None,
             False,
             True,
             0.09,
-            0.98,
+            0.981,
             ExitType.NONE,
             lambda current_profit, **kwargs: (
                 -0.1 if current_profit < 0.6 else -(current_profit * 2)
@@ -525,6 +537,19 @@ def test_min_roi_reached3(default_conf, fee) -> None:
             ExitType.NONE,
             lambda **kwargs: None,
         ),
+        # Error case - Returning inf.
+        (
+            0.05,
+            0.9,
+            ExitType.NONE,
+            None,
+            False,
+            True,
+            0.09,
+            0.9,
+            ExitType.NONE,
+            lambda **kwargs: math.inf,
+        ),
     ],
 )
 def test_ft_stoploss_reached(
@@ -552,6 +577,8 @@ def test_ft_stoploss_reached(
         exchange="binance",
         open_rate=1,
         liquidation_price=liq,
+        price_precision=4,
+        precision_mode=2,
     )
     trade.adjust_min_max_rates(trade.open_rate, trade.open_rate)
     strategy.trailing_stop = trailing
@@ -577,7 +604,7 @@ def test_ft_stoploss_reached(
         assert sl_flag.exit_flag is False
     else:
         assert sl_flag.exit_flag is True
-    assert round(trade.stop_loss, 2) == adjusted
+    assert round(trade.stop_loss, 3) == adjusted
     current_rate2 = trade.open_rate * (1 + profit2)
 
     sl_flag = strategy.ft_stoploss_reached(
@@ -593,7 +620,7 @@ def test_ft_stoploss_reached(
         assert sl_flag.exit_flag is False
     else:
         assert sl_flag.exit_flag is True
-    assert round(trade.stop_loss, 2) == adjusted2
+    assert round(trade.stop_loss, 3) == adjusted2
 
     strategy.custom_stoploss = original_stopvalue