Merge remote-tracking branch 'origin' into use-datasieve

.devcontainer/Dockerfile

@@ -1,11 +1,12 @@
-FROM freqtradeorg/freqtrade:develop
+FROM freqtradeorg/freqtrade:develop_freqairl
 
 USER root
 # Install dependencies
 COPY requirements-dev.txt /freqtrade/
 
 RUN apt-get update \
-    && apt-get -y install git mercurial sudo vim build-essential \
+    && apt-get -y install --no-install-recommends apt-utils dialog \
+    && apt-get -y install --no-install-recommends git sudo vim build-essential \
     && apt-get clean \
     && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \
     && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \

.devcontainer/devcontainer.json

@@ -19,23 +19,24 @@
     "postCreateCommand": "freqtrade create-userdir --userdir user_data/",
 
     "workspaceFolder": "/workspaces/freqtrade",
-
+    "customizations": {
-    "settings": {
+        "settings": {
-        "terminal.integrated.shell.linux": "/bin/bash",
+            "terminal.integrated.shell.linux": "/bin/bash",
-        "editor.insertSpaces": true,
+            "editor.insertSpaces": true,
-        "files.trimTrailingWhitespace": true,
+            "files.trimTrailingWhitespace": true,
-        "[markdown]": {
+            "[markdown]": {
-            "files.trimTrailingWhitespace": false,
+                "files.trimTrailingWhitespace": false,
+            },
+            "python.pythonPath": "/usr/local/bin/python",
         },
-        "python.pythonPath": "/usr/local/bin/python",
-    },
 
-    // Add the IDs of extensions you want installed when the container is created.
+        // Add the IDs of extensions you want installed when the container is created.
-    "extensions": [
+        "extensions": [
-        "ms-python.python",
+            "ms-python.python",
-        "ms-python.vscode-pylance",
+            "ms-python.vscode-pylance",
-        "davidanson.vscode-markdownlint",
+            "davidanson.vscode-markdownlint",
-        "ms-azuretools.vscode-docker",
+            "ms-azuretools.vscode-docker",
-        "vscode-icons-team.vscode-icons",
+            "vscode-icons-team.vscode-icons",
-    ],
+        ],
+    }
 }

.pre-commit-config.yaml

@@ -8,14 +8,14 @@ repos:
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.0.1"
+    rev: "v1.3.0"
     hooks:
       - id: mypy
         exclude: build_helpers
         additional_dependencies:
           - types-cachetools==5.3.0.5
           - types-filelock==3.2.7
-          - types-requests==2.30.0.0
+          - types-requests==2.31.0.1
           - types-tabulate==0.9.0.2
           - types-python-dateutil==2.8.19.13
           - SQLAlchemy==2.0.15
@@ -30,7 +30,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.0.263'
+    rev: 'v0.0.270'
     hooks:
       - id: ruff
 

docs/freqai.md

@@ -76,7 +76,7 @@ pip install -r requirements-freqai.txt
 
 ### Usage with docker
 
-If you are using docker, a dedicated tag with FreqAI dependencies is available as `:freqai`. As such - you can replace the image line in your docker compose file with `image: freqtradeorg/freqtrade:develop_freqai`. This image contains the regular FreqAI dependencies. Similar to native installs, Catboost will not be available on ARM based devices.
+If you are using docker, a dedicated tag with FreqAI dependencies is available as `:freqai`. As such - you can replace the image line in your docker compose file with `image: freqtradeorg/freqtrade:develop_freqai`. This image contains the regular FreqAI dependencies. Similar to native installs, Catboost will not be available on ARM based devices. If you would like to use PyTorch or Reinforcement learning, you should use the torch or RL tags, `image: freqtradeorg/freqtrade:develop_freqaitorch`, `image: freqtradeorg/freqtrade:develop_freqairl`.
 
 !!! note "docker-compose-freqai.yml"
    We do provide an explicit docker-compose file for this in `docker/docker-compose-freqai.yml` - which can be used via `docker compose -f docker/docker-compose-freqai.yml run ...` - or can be copied to replace the original docker file. This docker-compose file also contains a (disabled) section to enable GPU resources within docker containers. This obviously assumes the system has GPU resources available.

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.3.7
 mkdocs==1.4.3
-mkdocs-material==9.1.14
+mkdocs-material==9.1.15
 mdx_truly_sane_lists==1.3
 pymdown-extensions==10.0.1
 jinja2==3.1.2

docs/strategy-customization.md

@@ -342,16 +342,12 @@ The above configuration would therefore mean:
 
 The calculation does include fees.
 
-To disable ROI completely, set it to an insanely high number:
+To disable ROI completely, set it to an empty dictionary:
 
 ```python
-minimal_roi = {
+minimal_roi = {}
-    "0": 100
-}
 ```
 
-While technically not completely disabled, this would exit once the trade reaches 10000% Profit.
-
 To use times based on candle duration (timeframe), the following snippet can be handy.
 This will allow you to change the timeframe for the strategy, and ROI times will still be set as candles (e.g. after 3 candles ...)
 

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2023.5.dev'
+__version__ = '2023.6.dev'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/commands/list_commands.py

@@ -1,7 +1,7 @@
 import csv
 import logging
 import sys
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Union
 
 import rapidjson
 from colorama import Fore, Style
@@ -11,9 +11,10 @@ from tabulate import tabulate
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import market_is_active, validate_exchanges
+from freqtrade.exchange import list_available_exchanges, market_is_active
 from freqtrade.misc import parse_db_uri_for_logging, plural
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
+from freqtrade.types import ValidExchangesType
 
 
 logger = logging.getLogger(__name__)
@@ -25,18 +26,42 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
     :param args: Cli args from Arguments()
     :return: None
     """
-    exchanges = validate_exchanges(args['list_exchanges_all'])
+    exchanges = list_available_exchanges(args['list_exchanges_all'])
 
     if args['print_one_column']:
-        print('\n'.join([e[0] for e in exchanges]))
+        print('\n'.join([e['name'] for e in exchanges]))
     else:
+        headers = {
+            'name': 'Exchange name',
+            'valid': 'Valid',
+            'supported': 'Supported',
+            'trade_modes': 'Markets',
+            'comment': 'Reason',
+            }
+
+        def build_entry(exchange: ValidExchangesType, valid: bool):
+            valid_entry = {'valid': exchange['valid']} if valid else {}
+            result: Dict[str, Union[str, bool]] = {
+                'name': exchange['name'],
+                **valid_entry,
+                'supported': 'Official' if exchange['supported'] else '',
+                'trade_modes': ', '.join(
+                    (f"{a['margin_mode']} " if a['margin_mode'] else '') + a['trading_mode']
+                    for a in exchange['trade_modes']
+                ),
+                'comment': exchange['comment'],
+            }
+
+            return result
+
         if args['list_exchanges_all']:
             print("All exchanges supported by the ccxt library:")
+            exchanges = [build_entry(e, True) for e in exchanges]
         else:
             print("Exchanges available for Freqtrade:")
-            exchanges = [e for e in exchanges if e[1] is not False]
+            exchanges = [build_entry(e, False) for e in exchanges if e['valid'] is not False]
 
-        print(tabulate(exchanges, headers=['Exchange name', 'Valid', 'reason']))
+        print(tabulate(exchanges, headers=headers, ))
 
 
 def _print_objs_tabular(objs: List, print_colorized: bool) -> None:

freqtrade/configuration/config_validation.py

@@ -174,7 +174,7 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None:
         return
 
     for pl in conf.get('pairlists', [{'method': 'StaticPairList'}]):
-        if (pl.get('method') == 'StaticPairList'
+        if (isinstance(pl, dict) and pl.get('method') == 'StaticPairList'
                 and not conf.get('exchange', {}).get('pair_whitelist')):
             raise OperationalException("StaticPairList requires pair_whitelist to be set.")
 

freqtrade/constants.py

@@ -148,7 +148,6 @@ CONF_SCHEMA = {
             'patternProperties': {
                 '^[0-9.]+$': {'type': 'number'}
             },
-            'minProperties': 1
         },
         'amount_reserve_percent': {'type': 'number', 'minimum': 0.0, 'maximum': 0.5},
         'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True, 'minimum': -1},

freqtrade/exchange/__init__.py

@@ -13,11 +13,11 @@ from freqtrade.exchange.exchange_utils import (ROUND_DOWN, ROUND_UP, amount_to_c
                                                amount_to_contracts, amount_to_precision,
                                                available_exchanges, ccxt_exchanges,
                                                contracts_to_amount, date_minus_candles,
-                                               is_exchange_known_ccxt, market_is_active,
+                                               is_exchange_known_ccxt, list_available_exchanges,
-                                               price_to_precision, timeframe_to_minutes,
+                                               market_is_active, price_to_precision,
-                                               timeframe_to_msecs, timeframe_to_next_date,
+                                               timeframe_to_minutes, timeframe_to_msecs,
-                                               timeframe_to_prev_date, timeframe_to_seconds,
+                                               timeframe_to_next_date, timeframe_to_prev_date,
-                                               validate_exchange, validate_exchanges)
+                                               timeframe_to_seconds, validate_exchange)
 from freqtrade.exchange.gate import Gate
 from freqtrade.exchange.hitbtc import Hitbtc
 from freqtrade.exchange.huobi import Huobi

freqtrade/exchange/exchange.py

@@ -191,7 +191,7 @@ class Exchange:
 
         # Converts the interval provided in minutes in config to seconds
         self.markets_refresh_interval: int = exchange_conf.get(
-            "markets_refresh_interval", 60) * 60
+            "markets_refresh_interval", 60) * 60 * 1000
 
         if self.trading_mode != TradingMode.SPOT and load_leverage_tiers:
             self.fill_leverage_tiers()
@@ -1662,39 +1662,18 @@ class Exchange:
 
         price_side = self._get_price_side(side, is_short, conf_strategy)
 
-        price_side_word = price_side.capitalize()
-
         if conf_strategy.get('use_order_book', False):
 
             order_book_top = conf_strategy.get('order_book_top', 1)
             if order_book is None:
                 order_book = self.fetch_l2_order_book(pair, order_book_top)
-            logger.debug('order_book %s', order_book)
+            rate = self._get_rate_from_ob(pair, side, order_book, name, price_side,
-            # top 1 = index 0
+                                          order_book_top)
-            try:
-                obside: OBLiteral = 'bids' if price_side == 'bid' else 'asks'
-                rate = order_book[obside][order_book_top - 1][0]
-            except (IndexError, KeyError) as e:
-                logger.warning(
-                    f"{pair} - {name} Price at location {order_book_top} from orderbook "
-                    f"could not be determined. Orderbook: {order_book}"
-                )
-                raise PricingError from e
-            logger.debug(f"{pair} - {name} price from orderbook {price_side_word}"
-                         f"side - top {order_book_top} order book {side} rate {rate:.8f}")
         else:
-            logger.debug(f"Using Last {price_side_word} / Last Price")
+            logger.debug(f"Using Last {price_side.capitalize()} / Last Price")
             if ticker is None:
                 ticker = self.fetch_ticker(pair)
-            ticker_rate = ticker[price_side]
+            rate = self._get_rate_from_ticker(side, ticker, conf_strategy, price_side)
-            if ticker['last'] and ticker_rate:
-                if side == 'entry' and ticker_rate > ticker['last']:
-                    balance = conf_strategy.get('price_last_balance', 0.0)
-                    ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
-                elif side == 'exit' and ticker_rate < ticker['last']:
-                    balance = conf_strategy.get('price_last_balance', 0.0)
-                    ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last'])
-            rate = ticker_rate
 
         if rate is None:
             raise PricingError(f"{name}-Rate for {pair} was empty.")
@@ -1703,6 +1682,43 @@ class Exchange:
 
         return rate
 
+    def _get_rate_from_ticker(self, side: EntryExit, ticker: Ticker, conf_strategy: Dict[str, Any],
+                              price_side: BidAsk) -> Optional[float]:
+        """
+        Get rate from ticker.
+        """
+        ticker_rate = ticker[price_side]
+        if ticker['last'] and ticker_rate:
+            if side == 'entry' and ticker_rate > ticker['last']:
+                balance = conf_strategy.get('price_last_balance', 0.0)
+                ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
+            elif side == 'exit' and ticker_rate < ticker['last']:
+                balance = conf_strategy.get('price_last_balance', 0.0)
+                ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last'])
+        rate = ticker_rate
+        return rate
+
+    def _get_rate_from_ob(self, pair: str, side: EntryExit, order_book: OrderBook, name: str,
+                          price_side: BidAsk, order_book_top: int) -> float:
+        """
+        Get rate from orderbook
+        :raises: PricingError if rate could not be determined.
+        """
+        logger.debug('order_book %s', order_book)
+        # top 1 = index 0
+        try:
+            obside: OBLiteral = 'bids' if price_side == 'bid' else 'asks'
+            rate = order_book[obside][order_book_top - 1][0]
+        except (IndexError, KeyError) as e:
+            logger.warning(
+                    f"{pair} - {name} Price at location {order_book_top} from orderbook "
+                    f"could not be determined. Orderbook: {order_book}"
+                )
+            raise PricingError from e
+        logger.debug(f"{pair} - {name} price from orderbook {price_side.capitalize()}"
+                     f"side - top {order_book_top} order book {side} rate {rate:.8f}")
+        return rate
+
     def get_rates(self, pair: str, refresh: bool, is_short: bool) -> Tuple[float, float]:
         entry_rate = None
         exit_rate = None

freqtrade/exchange/exchange_utils.py

@@ -9,7 +9,9 @@ import ccxt
 from ccxt import (DECIMAL_PLACES, ROUND, ROUND_DOWN, ROUND_UP, SIGNIFICANT_DIGITS, TICK_SIZE,
                   TRUNCATE, decimal_to_precision)
 
-from freqtrade.exchange.common import BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED
+from freqtrade.exchange.common import (BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED,
+                                       SUPPORTED_EXCHANGES)
+from freqtrade.types import ValidExchangesType
 from freqtrade.util import FtPrecise
 from freqtrade.util.datetime_helpers import dt_from_ts, dt_ts
 
@@ -55,14 +57,41 @@ def validate_exchange(exchange: str) -> Tuple[bool, str]:
     return True, ''
 
 
-def validate_exchanges(all_exchanges: bool) -> List[Tuple[str, bool, str]]:
+def _build_exchange_list_entry(
+        exchange_name: str, exchangeClasses: Dict[str, Any]) -> ValidExchangesType:
+    valid, comment = validate_exchange(exchange_name)
+    result: ValidExchangesType = {
+        'name': exchange_name,
+        'valid': valid,
+        'supported': exchange_name.lower() in SUPPORTED_EXCHANGES,
+        'comment': comment,
+        'trade_modes': [{'trading_mode': 'spot', 'margin_mode': ''}],
+    }
+    if resolved := exchangeClasses.get(exchange_name.lower()):
+        supported_modes = [{'trading_mode': 'spot', 'margin_mode': ''}] + [
+            {'trading_mode': tm.value, 'margin_mode': mm.value}
+            for tm, mm in resolved['class']._supported_trading_mode_margin_pairs
+        ]
+        result.update({
+            'trade_modes': supported_modes,
+        })
+
+    return result
+
+
+def list_available_exchanges(all_exchanges: bool) -> List[ValidExchangesType]:
     """
     :return: List of tuples with exchangename, valid, reason.
     """
     exchanges = ccxt_exchanges() if all_exchanges else available_exchanges()
-    exchanges_valid = [
+    from freqtrade.resolvers.exchange_resolver import ExchangeResolver
-        (e, *validate_exchange(e)) for e in exchanges
+
+    subclassed = {e['name'].lower(): e for e in ExchangeResolver.search_all_objects({}, False)}
+
+    exchanges_valid: List[ValidExchangesType] = [
+        _build_exchange_list_entry(e, subclassed) for e in exchanges
     ]
+
     return exchanges_valid
 
 

freqtrade/exchange/gate.py

@@ -33,7 +33,6 @@ class Gate(Exchange):
     _ft_has_futures: Dict = {
         "needs_trading_fees": True,
         "marketOrderRequiresPrice": False,
-        "tickers_have_bid_ask": False,
         "fee_cost_in_contracts": False,  # Set explicitly to false for clarity
         "order_props_in_contracts": ['amount', 'filled', 'remaining'],
         "stop_price_type_field": "price_type",

freqtrade/freqtradebot.py

@@ -1302,6 +1302,10 @@ class FreqtradeBot(LoggingMixin):
                             f"(orderid:{order['id']}) in order to add another one ...")
 
                 self.cancel_stoploss_on_exchange(trade)
+                if not trade.is_open:
+                    logger.warning(
+                        f"Trade {trade} is closed, not creating trailing stoploss order.")
+                    return
 
                 # Create new stoploss order
                 if not self.create_stoploss_order(trade=trade, stop_price=stoploss_norm):

freqtrade/loggers/__init__.py

@@ -32,7 +32,7 @@ def _set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None:
         logging.INFO if verbosity <= 2 else logging.DEBUG
     )
     logging.getLogger('telegram').setLevel(logging.INFO)
-    logging.getLogger('httpx').setLevel(logging.INFO)
+    logging.getLogger('httpx').setLevel(logging.WARNING)
 
     logging.getLogger('werkzeug').setLevel(
         logging.ERROR if api_verbosity == 'error' else logging.INFO

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -74,7 +74,7 @@ class VolatilityFilter(IPairList):
         needed_pairs: ListPairsWithTimeframes = [
             (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache]
 
-        since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days - 1))
+        since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days))
         # Get all candles
         candles = {}
         if needed_pairs:
@@ -103,7 +103,7 @@ class VolatilityFilter(IPairList):
 
         result = False
         if daily_candles is not None and not daily_candles.empty:
-            returns = (np.log(daily_candles.close / daily_candles.close.shift(-1)))
+            returns = (np.log(daily_candles["close"].shift(1) / daily_candles["close"]))
             returns.fillna(0, inplace=True)
 
             volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days)

freqtrade/resolvers/exchange_resolver.py

@@ -2,7 +2,8 @@
 This module loads custom exchanges
 """
 import logging
-from typing import Optional
+from inspect import isclass
+from typing import Any, Dict, List, Optional
 
 import freqtrade.exchange as exchanges
 from freqtrade.constants import Config, ExchangeConfig
@@ -72,3 +73,26 @@ class ExchangeResolver(IResolver):
             f"Impossible to load Exchange '{exchange_name}'. This class does not exist "
             "or contains Python code errors."
         )
+
+    @classmethod
+    def search_all_objects(cls, config: Config, enum_failed: bool,
+                           recursive: bool = False) -> List[Dict[str, Any]]:
+        """
+        Searches for valid objects
+        :param config: Config object
+        :param enum_failed: If True, will return None for modules which fail.
+            Otherwise, failing modules are skipped.
+        :param recursive: Recursively walk directory tree searching for strategies
+        :return: List of dicts containing 'name', 'class' and 'location' entries
+        """
+        result = []
+        for exchange_name in dir(exchanges):
+            exchange = getattr(exchanges, exchange_name)
+            if isclass(exchange) and issubclass(exchange, Exchange):
+                result.append({
+                    'name': exchange_name,
+                    'class': exchange,
+                    'location': exchange.__module__,
+                    'location_rel: ': exchange.__module__.replace('freqtrade.', ''),
+                })
+        return result

freqtrade/resolvers/iresolver.py

@@ -41,7 +41,7 @@ class IResolver:
     object_type: Type[Any]
     object_type_str: str
     user_subdir: Optional[str] = None
-    initial_search_path: Optional[Path]
+    initial_search_path: Optional[Path] = None
     # Optional config setting containing a path (strategy_path, freqaimodel_path)
     extra_path: Optional[str] = None
 

freqtrade/rpc/api_server/api_backtest.py

@@ -16,14 +16,14 @@ from freqtrade.exchange.common import remove_exchange_credentials
 from freqtrade.misc import deep_merge_dicts
 from freqtrade.rpc.api_server.api_schemas import (BacktestHistoryEntry, BacktestRequest,
                                                   BacktestResponse)
-from freqtrade.rpc.api_server.deps import get_config, is_webserver_mode
+from freqtrade.rpc.api_server.deps import get_config
 from freqtrade.rpc.api_server.webserver_bgwork import ApiBG
 from freqtrade.rpc.rpc import RPCException
 
 
 logger = logging.getLogger(__name__)
 
-# Private API, protected by authentication
+# Private API, protected by authentication and webserver_mode dependency
 router = APIRouter()
 
 
@@ -102,7 +102,7 @@ def __run_backtest_bg(btconfig: Config):
 @router.post('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
 async def api_start_backtest(
         bt_settings: BacktestRequest, background_tasks: BackgroundTasks,
-        config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
+        config=Depends(get_config)):
     ApiBG.bt['bt_error'] = None
     """Start backtesting if not done so already"""
     if ApiBG.bgtask_running:
@@ -143,7 +143,7 @@ async def api_start_backtest(
 
 
 @router.get('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
-def api_get_backtest(ws_mode=Depends(is_webserver_mode)):
+def api_get_backtest():
     """
     Get backtesting result.
     Returns Result after backtesting has been ran.
@@ -188,7 +188,7 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)):
 
 
 @router.delete('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
-def api_delete_backtest(ws_mode=Depends(is_webserver_mode)):
+def api_delete_backtest():
     """Reset backtesting"""
     if ApiBG.bgtask_running:
         return {
@@ -215,7 +215,7 @@ def api_delete_backtest(ws_mode=Depends(is_webserver_mode)):
 
 
 @router.get('/backtest/abort', response_model=BacktestResponse, tags=['webserver', 'backtest'])
-def api_backtest_abort(ws_mode=Depends(is_webserver_mode)):
+def api_backtest_abort():
     if not ApiBG.bgtask_running:
         return {
             "status": "not_running",
@@ -236,15 +236,14 @@ def api_backtest_abort(ws_mode=Depends(is_webserver_mode)):
 
 @router.get('/backtest/history', response_model=List[BacktestHistoryEntry],
             tags=['webserver', 'backtest'])
-def api_backtest_history(config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
+def api_backtest_history(config=Depends(get_config)):
     # Get backtest result history, read from metadata files
     return get_backtest_resultlist(config['user_data_dir'] / 'backtest_results')
 
 
 @router.get('/backtest/history/result', response_model=BacktestResponse,
             tags=['webserver', 'backtest'])
-def api_backtest_history_result(filename: str, strategy: str, config=Depends(get_config),
+def api_backtest_history_result(filename: str, strategy: str, config=Depends(get_config)):
-                                ws_mode=Depends(is_webserver_mode)):
     # Get backtest result history, read from metadata files
     fn = config['user_data_dir'] / 'backtest_results' / filename
     results: Dict[str, Any] = {

freqtrade/rpc/api_server/api_schemas.py

@@ -5,6 +5,7 @@ from pydantic import BaseModel
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, IntOrInf
 from freqtrade.enums import OrderTypeValues, SignalDirection, TradingMode
+from freqtrade.types import ValidExchangesType
 
 
 class Ping(BaseModel):
@@ -396,6 +397,10 @@ class StrategyListResponse(BaseModel):
     strategies: List[str]
 
 
+class ExchangeListResponse(BaseModel):
+    exchanges: List[ValidExchangesType]
+
+
 class FreqAIModelListResponse(BaseModel):
     freqaimodels: List[str]
 

freqtrade/rpc/api_server/api_v1.py

@@ -12,7 +12,8 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.rpc import RPC
 from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload,
                                                   BlacklistResponse, Count, Daily,
-                                                  DeleteLockRequest, DeleteTrade, ForceEnterPayload,
+                                                  DeleteLockRequest, DeleteTrade,
+                                                  ExchangeListResponse, ForceEnterPayload,
                                                   ForceEnterResponse, ForceExitPayload,
                                                   FreqAIModelListResponse, Health, Locks, Logs,
                                                   OpenTradeSchema, PairHistory, PerformanceEntry,
@@ -46,7 +47,8 @@ logger = logging.getLogger(__name__)
 # 2.26: increase /balance output
 # 2.27: Add /trades/<id>/reload endpoint
 # 2.28: Switch reload endpoint to Post
-API_VERSION = 2.28
+# 2.29: Add /exchanges endpoint
+API_VERSION = 2.29
 
 # Public API, requires no auth.
 router_public = APIRouter()
@@ -312,6 +314,15 @@ def get_strategy(strategy: str, config=Depends(get_config)):
     }
 
 
+@router.get('/exchanges', response_model=ExchangeListResponse, tags=[])
+def list_exchanges(config=Depends(get_config)):
+    from freqtrade.exchange import list_available_exchanges
+    exchanges = list_available_exchanges(config)
+    return {
+        'exchanges': exchanges,
+    }
+
+
 @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai'])
 def list_freqaimodels(config=Depends(get_config)):
     from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver

freqtrade/rpc/api_server/deps.py

@@ -1,7 +1,7 @@
 from typing import Any, AsyncIterator, Dict, Optional
 from uuid import uuid4
 
-from fastapi import Depends
+from fastapi import Depends, HTTPException
 
 from freqtrade.enums import RunMode
 from freqtrade.persistence import Trade
@@ -57,5 +57,6 @@ def get_message_stream():
 
 def is_webserver_mode(config=Depends(get_config)):
     if config['runmode'] != RunMode.WEBSERVER:
-        raise RPCException('Bot is not in the correct state')
+        raise HTTPException(status_code=503,
+                            detail='Bot is not in the correct state.')
     return None

freqtrade/rpc/api_server/webserver.py

@@ -118,6 +118,7 @@ class ApiServer(RPCHandler):
         from freqtrade.rpc.api_server.api_v1 import router as api_v1
         from freqtrade.rpc.api_server.api_v1 import router_public as api_v1_public
         from freqtrade.rpc.api_server.api_ws import router as ws_router
+        from freqtrade.rpc.api_server.deps import is_webserver_mode
         from freqtrade.rpc.api_server.web_ui import router_ui
 
         app.include_router(api_v1_public, prefix="/api/v1")
@@ -126,7 +127,8 @@ class ApiServer(RPCHandler):
                            dependencies=[Depends(http_basic_or_jwt_token)],
                            )
         app.include_router(api_backtest, prefix="/api/v1",
-                           dependencies=[Depends(http_basic_or_jwt_token)],
+                           dependencies=[Depends(http_basic_or_jwt_token),
+                                         Depends(is_webserver_mode)],
                            )
         app.include_router(ws_router, prefix="/api/v1")
         app.include_router(router_login, prefix="/api/v1", tags=["auth"])

freqtrade/rpc/rpc.py

@@ -755,7 +755,7 @@ class RPC:
         return {'status': 'Reloaded from orders from exchange'}
 
     def __exec_force_exit(self, trade: Trade, ordertype: Optional[str],
-                          amount: Optional[float] = None) -> None:
+                          amount: Optional[float] = None) -> bool:
         # Check if there is there is an open order
         fully_canceled = False
         if trade.open_order_id:
@@ -770,6 +770,9 @@ class RPC:
                 self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_EXIT'])
 
         if not fully_canceled:
+            if trade.open_order_id is not None:
+                # Order cancellation failed, so we can't exit.
+                return False
             # Get current rate and execute sell
             current_rate = self._freqtrade.exchange.get_rate(
                 trade.pair, side='exit', is_short=trade.is_short, refresh=True)
@@ -790,6 +793,9 @@ class RPC:
                 trade, current_rate, exit_check, ordertype=order_type,
                 sub_trade_amt=sub_amount)
 
+            return True
+        return False
+
     def _rpc_force_exit(self, trade_id: str, ordertype: Optional[str] = None, *,
                         amount: Optional[float] = None) -> Dict[str, str]:
         """
@@ -802,12 +808,12 @@ class RPC:
 
         with self._freqtrade._exit_lock:
             if trade_id == 'all':
-                # Execute sell for all open orders
+                # Execute exit for all open orders
                 for trade in Trade.get_open_trades():
                     self.__exec_force_exit(trade, ordertype)
                 Trade.commit()
                 self._freqtrade.wallets.update()
-                return {'result': 'Created sell orders for all open trades.'}
+                return {'result': 'Created exit orders for all open trades.'}
 
             # Query for trade
             trade = Trade.get_trades(
@@ -817,10 +823,12 @@ class RPC:
                 logger.warning('force_exit: Invalid argument received')
                 raise RPCException('invalid argument')
 
-            self.__exec_force_exit(trade, ordertype, amount)
+            result = self.__exec_force_exit(trade, ordertype, amount)
             Trade.commit()
             self._freqtrade.wallets.update()
-            return {'result': f'Created sell order for trade {trade_id}.'}
+            if not result:
+                raise RPCException('Failed to exit trade.')
+            return {'result': f'Created exit order for trade {trade_id}.'}
 
     def _force_entry_validations(self, pair: str, order_side: SignalDirection):
         if not self._freqtrade.config.get('force_entry_enable', False):

freqtrade/strategy/interface.py

@@ -48,7 +48,7 @@ class IStrategy(ABC, HyperStrategyMixin):
 
     _ft_params_from_file: Dict
     # associated minimal roi
-    minimal_roi: Dict = {"0": 10.0}
+    minimal_roi: Dict = {}
 
     # associated stoploss
     stoploss: float
@@ -1085,6 +1085,11 @@ class IStrategy(ABC, HyperStrategyMixin):
         exits: List[ExitCheckTuple] = []
         current_rate = rate
         current_profit = trade.calc_profit_ratio(current_rate)
+        current_profit_best = current_profit
+        if low is not None or high is not None:
+            # Set current rate to high for backtesting ROI exits
+            current_rate_best = (low if trade.is_short else high) or rate
+            current_profit_best = trade.calc_profit_ratio(current_rate_best)
 
         trade.adjust_min_max_rates(high or current_rate, low or current_rate)
 
@@ -1093,20 +1098,13 @@ class IStrategy(ABC, HyperStrategyMixin):
                                                 current_profit=current_profit,
                                                 force_stoploss=force_stoploss, low=low, high=high)
 
-        # Set current rate to high for backtesting exits
-        current_rate = (low if trade.is_short else high) or rate
-        current_profit = trade.calc_profit_ratio(current_rate)
-
         # if enter signal and ignore_roi is set, we don't need to evaluate min_roi.
         roi_reached = (not (enter and self.ignore_roi_if_entry_signal)
-                       and self.min_roi_reached(trade=trade, current_profit=current_profit,
+                       and self.min_roi_reached(trade=trade, current_profit=current_profit_best,
                                                 current_time=current_time))
 
         exit_signal = ExitType.NONE
         custom_reason = ''
-        # use provided rate in backtesting, not high/low.
-        current_rate = rate
-        current_profit = trade.calc_profit_ratio(current_rate)
 
         if self.use_exit_signal:
             if exit_ and not enter:
@@ -1265,7 +1263,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         :return: minimal ROI entry value or None if none proper ROI entry was found.
         """
         # Get highest entry in ROI dict where key <= trade-duration
-        roi_list = list(filter(lambda x: x <= trade_dur, self.minimal_roi.keys()))
+        roi_list = [x for x in self.minimal_roi.keys() if x <= trade_dur]
         if not roi_list:
             return None, None
         roi_entry = max(roi_list)

freqtrade/types/__init__.py

@@ -0,0 +1 @@
+from freqtrade.types.valid_exchanges_type import ValidExchangesType  # noqa: F401

freqtrade/types/valid_exchanges_type.py

@@ -0,0 +1,17 @@
+# Used for list-exchanges
+from typing import List
+
+from typing_extensions import TypedDict
+
+
+class TradeModeType(TypedDict):
+    trading_mode: str
+    margin_mode: str
+
+
+class ValidExchangesType(TypedDict):
+    name: str
+    valid: bool
+    supported: bool
+    comment: str
+    trade_modes: List[TradeModeType]

requirements-dev.txt

@@ -7,12 +7,12 @@
 -r docs/requirements-docs.txt
 
 coveralls==3.3.1
-ruff==0.0.269
+ruff==0.0.270
 mypy==1.3.0
 pre-commit==3.3.2
 pytest==7.3.1
 pytest-asyncio==0.21.0
-pytest-cov==4.0.0
+pytest-cov==4.1.0
 pytest-mock==3.10.0
 pytest-random-order==1.1.0
 isort==5.12.0
@@ -25,6 +25,6 @@ nbconvert==7.4.0
 # mypy types
 types-cachetools==5.3.0.5
 types-filelock==3.2.7
-types-requests==2.30.0.0
+types-requests==2.31.0.1
 types-tabulate==0.9.0.2
 types-python-dateutil==2.8.19.13

requirements.txt

@@ -1,17 +1,17 @@
 numpy==1.24.3
-pandas==2.0.1
+pandas==2.0.2
 pandas-ta==0.3.14b
 
-ccxt==3.1.5
+ccxt==3.1.23
-cryptography==40.0.2; platform_machine != 'armv7l'
+cryptography==41.0.1; platform_machine != 'armv7l'
 cryptography==40.0.1; platform_machine == 'armv7l'
 aiohttp==3.8.4
 SQLAlchemy==2.0.15
 python-telegram-bot==20.3
 # can't be hard-pinned due to telegram-bot pinning httpx with ~
-httpx>=0.23.3
+httpx>=0.24.1
 arrow==1.2.3
-cachetools==5.3.0
+cachetools==5.3.1
 requests==2.31.0
 urllib3==2.0.2
 jsonschema==4.17.3
@@ -23,7 +23,7 @@ jinja2==3.1.2
 tables==3.8.0
 blosc==1.11.1
 joblib==1.2.0
-rich==13.3.5
+rich==13.4.1
 pyarrow==12.0.0; platform_machine != 'armv7l'
 
 # find first, C search in arrays
@@ -32,14 +32,14 @@ py_find_1st==1.1.5
 # Load ticker files 30% faster
 python-rapidjson==1.10
 # Properly format api responses
-orjson==3.8.12
+orjson==3.9.0
 
 # Notify systemd
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.95.2
+fastapi==0.96.0
-pydantic==1.10.7
+pydantic==1.10.8
 uvicorn==0.22.0
 pyjwt==2.7.0
 aiofiles==23.1.0

setup.py

@@ -107,7 +107,7 @@ setup(
         'ast-comments',
         'aiohttp',
         'cryptography',
-        'httpx',
+        'httpx>=0.24.1',
         'python-dateutil',
         'packaging',
     ],

tests/exchange/test_exchange.py

@@ -633,21 +633,23 @@ def test__load_markets(default_conf, mocker, caplog):
     assert ex.markets == expected_return
 
 
-def test_reload_markets(default_conf, mocker, caplog):
+def test_reload_markets(default_conf, mocker, caplog, time_machine):
     caplog.set_level(logging.DEBUG)
     initial_markets = {'ETH/BTC': {}}
     updated_markets = {'ETH/BTC': {}, "LTC/BTC": {}}
-
+    start_dt = dt_now()
+    time_machine.move_to(start_dt, tick=False)
     api_mock = MagicMock()
     api_mock.load_markets = MagicMock(return_value=initial_markets)
     default_conf['exchange']['markets_refresh_interval'] = 10
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance",
                                     mock_markets=False)
     exchange._load_async_markets = MagicMock()
-    exchange._last_markets_refresh = dt_ts()
+    assert exchange._last_markets_refresh == dt_ts()
 
     assert exchange.markets == initial_markets
 
+    time_machine.move_to(start_dt + timedelta(minutes=8), tick=False)
     # less than 10 minutes have passed, no reload
     exchange.reload_markets()
     assert exchange.markets == initial_markets
@@ -655,12 +657,18 @@ def test_reload_markets(default_conf, mocker, caplog):
 
     api_mock.load_markets = MagicMock(return_value=updated_markets)
     # more than 10 minutes have passed, reload is executed
-    exchange._last_markets_refresh = dt_ts(dt_now() - timedelta(minutes=15))
+    time_machine.move_to(start_dt + timedelta(minutes=11), tick=False)
     exchange.reload_markets()
     assert exchange.markets == updated_markets
     assert exchange._load_async_markets.call_count == 1
     assert log_has('Performing scheduled market reload..', caplog)
 
+    # Not called again
+    exchange._load_async_markets.reset_mock()
+
+    exchange.reload_markets()
+    assert exchange._load_async_markets.call_count == 0
+
 
 def test_reload_markets_exception(default_conf, mocker, caplog):
     caplog.set_level(logging.DEBUG)

tests/optimize/test_backtest_detail.py

@@ -820,7 +820,7 @@ tc52 = BTContainer(data=[
     [2, 4900, 5250, 4500, 5100, 6172, 0, 0],  # Order readjust
     [3, 5100, 5100, 4650, 4750, 6172, 0, 0],  # stoploss hit?
     [4, 4750, 4950, 4350, 4750, 6172, 0, 0]],
-    stop_loss=-0.03, roi={"0": 0.10}, profit_perc=-0.03,
+    stop_loss=-0.03, roi={}, profit_perc=-0.03,
     use_exit_signal=True, timeout=1000,
     custom_entry_price=4200, adjust_entry_price=5200,
     trades=[BTrade(exit_reason=ExitType.STOP_LOSS, open_tick=1, close_tick=2, is_short=False)]

tests/rpc/test_rpc.py

@@ -703,15 +703,15 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None:
         rpc._rpc_force_exit(None)
 
     msg = rpc._rpc_force_exit('all')
-    assert msg == {'result': 'Created sell orders for all open trades.'}
+    assert msg == {'result': 'Created exit orders for all open trades.'}
 
     freqtradebot.enter_positions()
     msg = rpc._rpc_force_exit('all')
-    assert msg == {'result': 'Created sell orders for all open trades.'}
+    assert msg == {'result': 'Created exit orders for all open trades.'}
 
     freqtradebot.enter_positions()
     msg = rpc._rpc_force_exit('2')
-    assert msg == {'result': 'Created sell order for trade 2.'}
+    assert msg == {'result': 'Created exit order for trade 2.'}
 
     freqtradebot.state = State.STOPPED
     with pytest.raises(RPCException, match=r'.*trader is not running*'):
@@ -761,27 +761,11 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None:
 
     freqtradebot.config['max_open_trades'] = 3
     freqtradebot.enter_positions()
-    trade = Trade.session.scalars(select(Trade).filter(Trade.id == '2')).first()
-    amount = trade.amount
-    # make an limit-buy open trade, if there is no 'filled', don't sell it
-    mocker.patch(
-        f'{EXMS}.fetch_order',
-        return_value={
-            'status': 'open',
-            'type': 'limit',
-            'side': 'buy',
-            'filled': None
-        }
-    )
-    # check that the trade is called, which is done by ensuring exchange.cancel_order is called
-    msg = rpc._rpc_force_exit('4')
-    assert msg == {'result': 'Created sell order for trade 4.'}
-    assert cancel_order_mock.call_count == 2
-    assert trade.amount == amount
 
+    cancel_order_mock.reset_mock()
     trade = Trade.session.scalars(select(Trade).filter(Trade.id == '3')).first()
-
+    amount = trade.amount
-    # make an limit-sell open trade
+    # make an limit-sell open order trade
     mocker.patch(
         f'{EXMS}.fetch_order',
         return_value={
@@ -794,10 +778,54 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None:
             'id': trade.orders[0].order_id,
         }
     )
+    cancel_order_3 = mocker.patch(
+        f'{EXMS}.cancel_order_with_result',
+        return_value={
+            'status': 'canceled',
+            'type': 'limit',
+            'side': 'sell',
+            'amount': amount,
+            'remaining': amount,
+            'filled': 0.0,
+            'id': trade.orders[0].order_id,
+        }
+    )
     msg = rpc._rpc_force_exit('3')
-    assert msg == {'result': 'Created sell order for trade 3.'}
+    assert msg == {'result': 'Created exit order for trade 3.'}
     # status quo, no exchange calls
-    assert cancel_order_mock.call_count == 3
+    assert cancel_order_3.call_count == 1
+    assert cancel_order_mock.call_count == 0
+
+    trade = Trade.session.scalars(select(Trade).filter(Trade.id == '2')).first()
+    amount = trade.amount
+    # make an limit-buy open trade, if there is no 'filled', don't sell it
+    mocker.patch(
+        f'{EXMS}.fetch_order',
+        return_value={
+            'status': 'open',
+            'type': 'limit',
+            'side': 'buy',
+            'filled': None
+        }
+    )
+    cancel_order_4 = mocker.patch(
+            f'{EXMS}.cancel_order_with_result',
+            return_value={
+                'status': 'canceled',
+                'type': 'limit',
+                'side': 'sell',
+                'amount': amount,
+                'remaining': 0.0,
+                'filled': amount,
+                'id': trade.orders[0].order_id,
+                }
+            )
+    # check that the trade is called, which is done by ensuring exchange.cancel_order is called
+    msg = rpc._rpc_force_exit('4')
+    assert msg == {'result': 'Created exit order for trade 4.'}
+    assert cancel_order_4.call_count == 1
+    assert cancel_order_mock.call_count == 0
+    assert trade.amount == amount
 
 
 def test_performance_handle(default_conf_usdt, ticker, fee, mocker) -> None:

tests/rpc/test_rpc_apiserver.py

@@ -1333,7 +1333,7 @@ def test_api_forceexit(botclient, mocker, ticker, fee, markets):
     rc = client_post(client, f"{BASE_URI}/forceexit",
                      data={"tradeid": "5", "ordertype": "market", "amount": 23})
     assert_response(rc)
-    assert rc.json() == {'result': 'Created sell order for trade 5.'}
+    assert rc.json() == {'result': 'Created exit order for trade 5.'}
     Trade.rollback()
 
     trade = Trade.get_trades([Trade.id == 5]).first()
@@ -1343,7 +1343,7 @@ def test_api_forceexit(botclient, mocker, ticker, fee, markets):
     rc = client_post(client, f"{BASE_URI}/forceexit",
                      data={"tradeid": "5"})
     assert_response(rc)
-    assert rc.json() == {'result': 'Created sell order for trade 5.'}
+    assert rc.json() == {'result': 'Created exit order for trade 5.'}
     Trade.rollback()
 
     trade = Trade.get_trades([Trade.id == 5]).first()
@@ -1578,6 +1578,47 @@ def test_api_strategy(botclient):
     assert_response(rc, 500)
 
 
+def test_api_exchanges(botclient):
+    ftbot, client = botclient
+
+    rc = client_get(client, f"{BASE_URI}/exchanges")
+    assert_response(rc)
+    response = rc.json()
+    assert isinstance(response['exchanges'], list)
+    assert len(response['exchanges']) > 20
+    okx = [x for x in response['exchanges'] if x['name'] == 'okx'][0]
+    assert okx == {
+        "name": "okx",
+        "valid": True,
+        "supported": True,
+        "comment": "",
+        "trade_modes": [
+            {
+                "trading_mode": "spot",
+                "margin_mode": ""
+            },
+            {
+                "trading_mode": "futures",
+                "margin_mode": "isolated"
+            }
+        ]
+    }
+
+    mexc = [x for x in response['exchanges'] if x['name'] == 'mexc'][0]
+    assert mexc == {
+        "name": "mexc",
+        "valid": True,
+        "supported": False,
+        "comment": "",
+        "trade_modes": [
+                {
+                    "trading_mode": "spot",
+                    "margin_mode": ""
+                }
+            ]
+    }
+
+
 def test_api_freqaimodels(botclient, tmpdir, mocker):
     ftbot, client = botclient
     ftbot.config['user_data_dir'] = Path(tmpdir)
@@ -1673,7 +1714,8 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
 
         rc = client_get(client, f"{BASE_URI}/backtest")
         # Backtest prevented in default mode
-        assert_response(rc, 502)
+        assert_response(rc, 503)
+        assert rc.json()['detail'] == 'Bot is not in the correct state.'
 
         ftbot.config['runmode'] = RunMode.WEBSERVER
         # Backtesting not started yet
@@ -1812,7 +1854,9 @@ def test_api_backtest_history(botclient, mocker, testdatadir):
                      ])
 
     rc = client_get(client, f"{BASE_URI}/backtest/history")
-    assert_response(rc, 502)
+    assert_response(rc, 503)
+    assert rc.json()['detail'] == 'Bot is not in the correct state.'
+
     ftbot.config['user_data_dir'] = testdatadir
     ftbot.config['runmode'] = RunMode.WEBSERVER
 
@@ -1930,3 +1974,4 @@ def test_api_ws_send_msg(default_conf, mocker, caplog):
 
     finally:
         ApiServer.shutdown()
+        ApiServer.shutdown()

tests/test_main.py

@@ -118,7 +118,7 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None:
 def test_main_operational_exception1(mocker, default_conf, caplog) -> None:
     patch_exchange(mocker)
     mocker.patch(
-        'freqtrade.commands.list_commands.validate_exchanges',
+        'freqtrade.commands.list_commands.list_available_exchanges',
         MagicMock(side_effect=ValueError('Oh snap!'))
     )
     patched_configuration_load_config_file(mocker, default_conf)
@@ -132,7 +132,7 @@ def test_main_operational_exception1(mocker, default_conf, caplog) -> None:
     assert log_has('Fatal exception!', caplog)
     assert not log_has_re(r'SIGINT.*', caplog)
     mocker.patch(
-        'freqtrade.commands.list_commands.validate_exchanges',
+        'freqtrade.commands.list_commands.list_available_exchanges',
         MagicMock(side_effect=KeyboardInterrupt)
     )
     with pytest.raises(SystemExit):