Merge remote-tracking branch 'upstream/develop' into feature/fetch-public-trades

.github/workflows/ci.yml

@@ -55,7 +55,7 @@ jobs:
 
     - name: Installation - *nix
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -192,7 +192,7 @@ jobs:
 
     - name: Installation (python)
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -422,7 +422,7 @@ jobs:
 
     - name: Installation - *nix
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include

.pre-commit-config.yaml

@@ -16,10 +16,10 @@ repos:
         additional_dependencies:
           - types-cachetools==5.3.0.7
           - types-filelock==3.2.7
-          - types-requests==2.32.0.20240602
+          - types-requests==2.32.0.20240622
           - types-tabulate==0.9.0.20240106
           - types-python-dateutil==2.9.0.20240316
-          - SQLAlchemy==2.0.30
+          - SQLAlchemy==2.0.31
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.4.9'
+    rev: 'v0.4.10'
     hooks:
       - id: ruff
 

Dockerfile

@@ -25,7 +25,7 @@ FROM base as python-deps
 RUN  apt-get update \
   && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \
   && apt-get clean \
-  && pip install --upgrade pip wheel
+  && pip install --upgrade "pip<=24.0" wheel
 
 # Install TA-lib
 COPY build_helpers/* /tmp/
@@ -35,7 +35,7 @@ ENV LD_LIBRARY_PATH /usr/local/lib
 # Install dependencies
 COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/
 USER ftuser
-RUN  pip install --user --no-cache-dir numpy \
+RUN  pip install --user --no-cache-dir "numpy<2.0" \
   && pip install --user --no-cache-dir -r requirements-hyperopt.txt
 
 # Copy dependencies to runtime-image

build_helpers/install_windows.ps1

@@ -1,6 +1,6 @@
 # vendored Wheels compiled via https://github.com/xmatthias/ta-lib-python/tree/ta_bundled_040
 
-python -m pip install --upgrade pip wheel
+python -m pip install --upgrade "pip<=24.0" wheel
 
 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
 

docker/Dockerfile.armhf

@@ -17,7 +17,7 @@ RUN mkdir /freqtrade \
   && chown ftuser:ftuser /freqtrade \
   # Allow sudoers
   && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers \
-  && pip install --upgrade pip
+  && pip install --upgrade "pip<=24.0"
 
 WORKDIR /freqtrade
 

docs/includes/pairlists.md

@@ -373,7 +373,7 @@ Filters low-value coins which would not allow setting stoplosses.
 
 Namely, pairs are blacklisted if a variance of one percent or more in the stop price would be caused by precision rounding on the exchange, i.e. `rounded(stop_price) <= rounded(stop_price * 0.99)`. The idea is to avoid coins with a value VERY close to their lower trading boundary, not allowing setting of proper stoploss.
 
-!!! Tip "PerformanceFilter is pointless for futures trading"
+!!! Tip "PrecisionFilter is pointless for futures trading"
     The above does not apply to shorts. And for longs, in theory the trade will be liquidated first.
 
 !!! Warning "Backtesting"

docs/plotting.md

@@ -2,6 +2,14 @@
 
 This page explains how to plot prices, indicators and profits.
 
+!!! Warning "Deprecated"
+    The commands described in this page (`plot-dataframe`, `plot-profit`) should be considered deprecated and are in maintenance mode.
+    This is mostly for the performance problems even medium sized plots can cause, but also because "store a file and open it in a browser" isn't very intuitive from a UI perspective.
+
+    While there are no immediate plans to remove them, they are not actively maintained - and may be removed short-term should major changes be required to keep them working.
+    
+    Please use [FreqUI](freq-ui.md) for plotting needs, which doesn't struggle with the same performance problems.
+
 ## Installation / Setup
 
 Plotting modules use the Plotly library. You can install / upgrade this by running the following command:

docs/strategy-callbacks.md

@@ -165,7 +165,9 @@ E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoplo
 During backtesting, `current_rate` (and `current_profit`) are provided against the candle's high (or low for short trades) - while the resulting stoploss is evaluated against the candle's low (or high for short trades).
 
 The absolute value of the return value is used (the sign is ignored), so returning `0.05` or `-0.05` have the same result, a stoploss 5% below the current price.
-Returning None will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+Returning `None` will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+`NaN` and `inf` values are considered invalid and will be ignored (identical to `None`).
+
 
 Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchangefreqtrade)).
 

freqtrade/data/history/history_utils.py

@@ -618,6 +618,11 @@ def download_data_main(config: Config) -> None:
     # Start downloading
     try:
         if config.get("download_trades"):
+            if not exchange.get_option("trades_has_history", True):
+                raise OperationalException(
+                    f"Trade history not available for {exchange.name}. "
+                    "You cannot use --dl-trades for this exchange."
+                )
             pairs_not_available = refresh_backtest_trades_data(
                 exchange,
                 pairs=expanded_pairs,

freqtrade/exchange/binance.py

@@ -28,6 +28,7 @@ class Binance(Exchange):
         "ohlcv_candle_limit": 1000,
         "trades_pagination": "id",
         "trades_pagination_arg": "fromId",
+        "trades_has_history": True,
         "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000],
     }
     _ft_has_futures: Dict = {

freqtrade/exchange/bingx.py

@@ -20,4 +20,5 @@ class Bingx(Exchange):
         "stoploss_on_exchange": True,
         "stoploss_order_types": {"limit": "limit", "market": "market"},
         "order_time_in_force": ["GTC", "IOC", "PO"],
+        "trades_has_history": False,  # Endpoint doesn't seem to support pagination
     }

freqtrade/exchange/bitmart.py

@@ -18,4 +18,5 @@ class Bitmart(Exchange):
     _ft_has: Dict = {
         "stoploss_on_exchange": False,  # Bitmart API does not support stoploss orders
         "ohlcv_candle_limit": 200,
+        "trades_has_history": False,  # Endpoint doesn't seem to support pagination
     }

freqtrade/exchange/bybit.py

@@ -33,6 +33,7 @@ class Bybit(Exchange):
         "ohlcv_candle_limit": 1000,
         "ohlcv_has_history": True,
         "order_time_in_force": ["GTC", "FOK", "IOC", "PO"],
+        "trades_has_history": False,  # Endpoint doesn't support pagination
     }
     _ft_has_futures: Dict = {
         "ohlcv_has_history": True,

freqtrade/exchange/exchange.py

@@ -125,6 +125,7 @@ class Exchange:
         "trades_limit": 1000,  # Limit for 1 call to fetch_trades
         "trades_pagination": "time",  # Possible are "time" or "id"
         "trades_pagination_arg": "since",
+        "trades_has_history": False,
         "l2_limit_range": None,
         "l2_limit_range_required": True,  # Allow Empty L2 limit (kucoin)
         "mark_ohlcv_price": "mark",

freqtrade/exchange/gate.py

@@ -31,6 +31,7 @@ class Gate(Exchange):
         "stop_price_param": "stopPrice",
         "stop_price_prop": "stopPrice",
         "marketOrderRequiresPrice": True,
+        "trades_has_history": False,  # Endpoint would support this - but ccxt doesn't.
     }
 
     _ft_has_futures: Dict = {

freqtrade/exchange/htx.py

@@ -28,6 +28,7 @@ class Htx(Exchange):
             "1w": 500,
             "1M": 500,
         },
+        "trades_has_history": False,  # Endpoint doesn't have a "since" parameter
     }
 
     def _get_stop_params(self, side: BuySell, ordertype: str, stop_price: float) -> Dict:

freqtrade/exchange/kraken.py

@@ -31,6 +31,7 @@ class Kraken(Exchange):
         "trades_pagination": "id",
         "trades_pagination_arg": "since",
         "trades_pagination_overlap": False,
+        "trades_has_history": True,
         "mark_ohlcv_timeframe": "4h",
     }
 

freqtrade/exchange/okx.py

@@ -33,6 +33,7 @@ class Okx(Exchange):
         "funding_fee_timeframe": "8h",
         "stoploss_order_types": {"limit": "limit"},
         "stoploss_on_exchange": True,
+        "trades_has_history": False,  # Endpoint doesn't have a "since" parameter
     }
     _ft_has_futures: Dict = {
         "tickers_have_quoteVolume": False,

freqtrade/strategy/interface.py

@@ -6,6 +6,7 @@ This module defines the interface to apply for strategies
 import logging
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta, timezone
+from math import isinf, isnan
 from typing import Dict, List, Optional, Tuple, Union
 
 from pandas import DataFrame
@@ -1425,7 +1426,9 @@ class IStrategy(ABC, HyperStrategyMixin):
                 after_fill=after_fill,
             )
             # Sanity check - error cases will return None
-            if stop_loss_value_custom:
+            if stop_loss_value_custom and not (
+                isnan(stop_loss_value_custom) or isinf(stop_loss_value_custom)
+            ):
                 stop_loss_value = stop_loss_value_custom
                 trade.adjust_stop_loss(
                     bound or current_rate, stop_loss_value, allow_refresh=after_fill

requirements-dev.txt

@@ -7,7 +7,7 @@
 -r docs/requirements-docs.txt
 
 coveralls==4.0.1
-ruff==0.4.9
+ruff==0.4.10
 mypy==1.10.0
 pre-commit==3.7.1
 pytest==8.2.2
@@ -26,6 +26,6 @@ nbconvert==7.16.4
 # mypy types
 types-cachetools==5.3.0.7
 types-filelock==3.2.7
-types-requests==2.32.0.20240602
+types-requests==2.32.0.20240622
 types-tabulate==0.9.0.20240106
 types-python-dateutil==2.9.0.20240316

requirements-hyperopt.txt

@@ -5,4 +5,4 @@
 scipy==1.13.1
 scikit-learn==1.5.0
 ft-scikit-optimize==0.9.2
-filelock==3.15.1
+filelock==3.15.4

requirements.txt

@@ -1,13 +1,13 @@
 numpy==1.26.4
 pandas==2.2.2
-bottleneck==1.3.8
+bottleneck==1.4.0
-numexpr==2.10.0
+numexpr==2.10.1
 pandas-ta==0.3.14b
 
-ccxt==4.3.46
+ccxt==4.3.50
 cryptography==42.0.8
 aiohttp==3.9.5
-SQLAlchemy==2.0.30
+SQLAlchemy==2.0.31
 python-telegram-bot==21.3
 # can't be hard-pinned due to telegram-bot pinning httpx with ~
 httpx>=0.24.1
@@ -43,7 +43,7 @@ pydantic==2.7.4
 uvicorn==0.30.1
 pyjwt==2.8.0
 aiofiles==23.2.1
-psutil==5.9.8
+psutil==6.0.0
 
 # Support for colorized terminal output
 colorama==0.4.6

setup.sh

@@ -49,7 +49,7 @@ function updateenv() {
     source .venv/bin/activate
     SYS_ARCH=$(uname -m)
     echo "pip install in-progress. Please wait..."
-    ${PYTHON} -m pip install --upgrade pip wheel setuptools
+    ${PYTHON} -m pip install --upgrade "pip<=24.0" wheel setuptools
     REQUIREMENTS_HYPEROPT=""
     REQUIREMENTS_PLOT=""
     REQUIREMENTS_FREQAI=""

tests/data/test_download_data.py

@@ -83,6 +83,12 @@ def test_download_data_main_trades(mocker):
     assert dl_mock.call_count == 1
     assert convert_mock.call_count == 1
 
+    # Exchange that doesn't support historic downloads
+    config["exchange"]["name"] = "bybit"
+    with pytest.raises(OperationalException, match=r"Trade history not available for .*"):
+        config
+        download_data_main(config)
+
 
 def test_download_data_main_data_invalid(mocker):
     patch_exchange(mocker, id="kraken")

tests/strategy/test_interface.py

@@ -1,5 +1,6 @@
 # pragma pylint: disable=missing-docstring, C0103
 import logging
+import math
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from unittest.mock import MagicMock
@@ -458,55 +459,66 @@ def test_min_roi_reached3(default_conf, fee) -> None:
             ExitType.TRAILING_STOP_LOSS,
             None,
         ),
-        (0.01, 0.96, ExitType.NONE, None, True, False, 0.05, 1, ExitType.NONE, None),
+        (0.01, 0.96, ExitType.NONE, None, True, False, 0.05, 0.998, ExitType.NONE, None),
-        (0.05, 1, ExitType.NONE, None, True, False, -0.01, 1, ExitType.TRAILING_STOP_LOSS, None),
+        (
+            0.05,
+            0.998,
+            ExitType.NONE,
+            None,
+            True,
+            False,
+            -0.01,
+            0.998,
+            ExitType.TRAILING_STOP_LOSS,
+            None,
+        ),
         # Default custom case - trails with 10%
-        (0.05, 0.95, ExitType.NONE, None, False, True, -0.02, 0.95, ExitType.NONE, None),
+        (0.05, 0.945, ExitType.NONE, None, False, True, -0.02, 0.945, ExitType.NONE, None),
         (
             0.05,
-            0.95,
+            0.945,
             ExitType.NONE,
             None,
             False,
             True,
             -0.06,
-            0.95,
+            0.945,
             ExitType.TRAILING_STOP_LOSS,
             None,
         ),
         (
             0.05,
-            1,
+            0.998,
             ExitType.NONE,
             None,
             False,
             True,
             -0.06,
-            1,
+            0.998,
             ExitType.TRAILING_STOP_LOSS,
             lambda **kwargs: -0.05,
         ),
         (
             0.05,
-            1,
+            0.998,
             ExitType.NONE,
             None,
             False,
             True,
             0.09,
-            1.04,
+            1.036,
             ExitType.NONE,
             lambda **kwargs: -0.05,
         ),
         (
             0.05,
-            0.95,
+            0.945,
             ExitType.NONE,
             None,
             False,
             True,
             0.09,
-            0.98,
+            0.981,
             ExitType.NONE,
             lambda current_profit, **kwargs: (
                 -0.1 if current_profit < 0.6 else -(current_profit * 2)
@@ -525,6 +537,19 @@ def test_min_roi_reached3(default_conf, fee) -> None:
             ExitType.NONE,
             lambda **kwargs: None,
         ),
+        # Error case - Returning inf.
+        (
+            0.05,
+            0.9,
+            ExitType.NONE,
+            None,
+            False,
+            True,
+            0.09,
+            0.9,
+            ExitType.NONE,
+            lambda **kwargs: math.inf,
+        ),
     ],
 )
 def test_ft_stoploss_reached(
@@ -552,6 +577,8 @@ def test_ft_stoploss_reached(
         exchange="binance",
         open_rate=1,
         liquidation_price=liq,
+        price_precision=4,
+        precision_mode=2,
     )
     trade.adjust_min_max_rates(trade.open_rate, trade.open_rate)
     strategy.trailing_stop = trailing
@@ -577,7 +604,7 @@ def test_ft_stoploss_reached(
         assert sl_flag.exit_flag is False
     else:
         assert sl_flag.exit_flag is True
-    assert round(trade.stop_loss, 2) == adjusted
+    assert round(trade.stop_loss, 3) == adjusted
     current_rate2 = trade.open_rate * (1 + profit2)
 
     sl_flag = strategy.ft_stoploss_reached(
@@ -593,7 +620,7 @@ def test_ft_stoploss_reached(
         assert sl_flag.exit_flag is False
     else:
         assert sl_flag.exit_flag is True
-    assert round(trade.stop_loss, 2) == adjusted2
+    assert round(trade.stop_loss, 3) == adjusted2
 
     strategy.custom_stoploss = original_stopvalue