Merge branch 'freqtrade:develop' into pixeebot/drip-2023-11-14-pixee-python/harden-pyyaml

.github/workflows/ci.yml

@@ -129,7 +129,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ "macos-latest", "macos-13", "macos-14" ]
+        os: [ "macos-12", "macos-13", "macos-14" ]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
         exclude:
           - os: "macos-14"

docs/rest-api.md

@@ -89,7 +89,8 @@ Make sure that the following 2 lines are available in your docker-compose file:
 ```
 
 !!! Danger "Security warning"
-    By using `8080:8080` in the docker port mapping, the API will be available to everyone connecting to the server under the correct port, so others may be able to control your bot.
+    By using `"8080:8080"` (or `"0.0.0.0:8080:8080"`) in the docker port mapping, the API will be available to everyone connecting to the server under the correct port, so others may be able to control your bot.
+    This **may** be safe if you're running the bot in a secure environment (like your home network), but it's not recommended to expose the API to the internet.
 
 ## Rest API
 

freqtrade/data/history/datahandlers/idatahandler.py

@@ -302,8 +302,8 @@ class IDataHandler(ABC):
         Rebuild pair name from filename
         Assumes a asset name of max. 7 length to also support BTC-PERP and BTC-PERP:USD names.
         """
-        res = re.sub(r'^(([A-Za-z\d]{1,10})|^([A-Za-z\-]{1,6}))(_)', r'\g<1>/', pair, 1)
-        res = re.sub('_', ':', res, 1)
+        res = re.sub(r'^(([A-Za-z\d]{1,10})|^([A-Za-z\-]{1,6}))(_)', r'\g<1>/', pair, count=1)
+        res = re.sub('_', ':', res, count=1)
         return res
 
     def ohlcv_load(self, pair, timeframe: str,

freqtrade/exchange/__init__.py

@@ -25,6 +25,7 @@ from freqtrade.exchange.exchange_utils_timeframe import (timeframe_to_minutes, t
 from freqtrade.exchange.gate import Gate
 from freqtrade.exchange.hitbtc import Hitbtc
 from freqtrade.exchange.htx import Htx
+from freqtrade.exchange.idex import Idex
 from freqtrade.exchange.kraken import Kraken
 from freqtrade.exchange.kucoin import Kucoin
 from freqtrade.exchange.okx import Okx

freqtrade/exchange/idex.py

@@ -0,0 +1,19 @@
+""" Idex exchange subclass """
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Idex(Exchange):
+    """
+    Idex exchange class. Contains adjustments needed for Freqtrade to work
+    with this exchange.
+    """
+
+    _ft_has: Dict = {
+        "ohlcv_candle_limit": 1000,
+    }

freqtrade/freqtradebot.py

@@ -462,6 +462,7 @@ class FreqtradeBot(LoggingMixin):
                 trade.pair, trade.open_date_utc - timedelta(seconds=10))
             prev_exit_reason = trade.exit_reason
             prev_trade_state = trade.is_open
+            prev_trade_amount = trade.amount
             for order in orders:
                 trade_order = [o for o in trade.orders if o.order_id == order['id']]
 
@@ -493,6 +494,26 @@ class FreqtradeBot(LoggingMixin):
                                         send_msg=prev_trade_state != trade.is_open)
             else:
                 trade.exit_reason = prev_exit_reason
+                total = self.wallets.get_total(trade.base_currency) if trade.base_currency else 0
+                if total < trade.amount:
+                    if total > trade.amount * 0.98:
+                        logger.warning(
+                            f"{trade} has a total of {trade.amount} {trade.base_currency}, "
+                            f"but the Wallet shows a total of {total} {trade.base_currency}. "
+                            f"Adjusting trade amount to {total}."
+                            "This may however lead to further issues."
+                        )
+                        trade.amount = total
+                    else:
+                        logger.warning(
+                            f"{trade} has a total of {trade.amount} {trade.base_currency}, "
+                            f"but the Wallet shows a total of {total} {trade.base_currency}. "
+                            "Refusing to adjust as the difference is too large."
+                            "This may however lead to further issues."
+                        )
+                if prev_trade_amount != trade.amount:
+                    # Cancel stoploss on exchange if the amount changed
+                    trade = self.cancel_stoploss_on_exchange(trade)
             Trade.commit()
 
         except ExchangeError:
@@ -1948,21 +1969,23 @@ class FreqtradeBot(LoggingMixin):
 
         trade.update_trade(order_obj, not send_msg)
 
-        trade = self._update_trade_after_fill(trade, order_obj)
+        trade = self._update_trade_after_fill(trade, order_obj, send_msg)
         Trade.commit()
 
         self.order_close_notify(trade, order_obj, stoploss_order, send_msg)
 
         return False
 
-    def _update_trade_after_fill(self, trade: Trade, order: Order) -> Trade:
+    def _update_trade_after_fill(self, trade: Trade, order: Order, send_msg: bool) -> Trade:
         if order.status in constants.NON_OPEN_EXCHANGE_STATES:
             strategy_safe_wrapper(
                 self.strategy.order_filled, default_retval=None)(
                 pair=trade.pair, trade=trade, order=order, current_time=datetime.now(timezone.utc))
             # If a entry order was closed, force update on stoploss on exchange
             if order.ft_order_side == trade.entry_side:
-                trade = self.cancel_stoploss_on_exchange(trade)
+                if send_msg:
+                    # Don't cancel stoploss in recovery modes immediately
+                    trade = self.cancel_stoploss_on_exchange(trade)
                 if not self.edge:
                     # TODO: should shorting/leverage be supported by Edge,
                     # then this will need to be fixed.

freqtrade/plot/plotting.py

@@ -440,8 +440,8 @@ def create_scatter(
 
 def generate_candlestick_graph(
         pair: str, data: pd.DataFrame, trades: Optional[pd.DataFrame] = None, *,
-        indicators1: List[str] = [], indicators2: List[str] = [],
-        plot_config: Dict[str, Dict] = {},
+        indicators1: Optional[List[str]] = None, indicators2: Optional[List[str]] = None,
+        plot_config: Optional[Dict[str, Dict]] = None,
         ) -> go.Figure:
     """
     Generate the graph from the data generated by Backtesting or from DB
@@ -454,7 +454,11 @@ def generate_candlestick_graph(
     :param plot_config: Dict of Dicts containing advanced plot configuration
     :return: Plotly figure
     """
-    plot_config = create_plotconfig(indicators1, indicators2, plot_config)
+    plot_config = create_plotconfig(
+        indicators1 or [],
+        indicators2 or [],
+        plot_config or {},
+    )
     rows = 2 + len(plot_config['subplots'])
     row_widths = [1 for _ in plot_config['subplots']]
     # Define the graph

freqtrade/plugins/pairlist/MarketCapPairList.py

@@ -38,7 +38,7 @@ class MarketCapPairList(IPairList):
         self._refresh_period = self._pairlistconfig.get('refresh_period', 86400)
         self._marketcap_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
         self._def_candletype = self._config['candle_type_def']
-        self._coingekko: CoinGeckoAPI = CoinGeckoAPI()
+        self._coingecko: CoinGeckoAPI = CoinGeckoAPI()
 
         if self._max_rank > 250:
             raise OperationalException(
@@ -127,7 +127,7 @@ class MarketCapPairList(IPairList):
         marketcap_list = self._marketcap_cache.get('marketcap')
 
         if marketcap_list is None:
-            data = self._coingekko.get_coins_markets(vs_currency='usd', order='market_cap_desc',
+            data = self._coingecko.get_coins_markets(vs_currency='usd', order='market_cap_desc',
                                                      per_page='250', page='1', sparkline='false',
                                                      locale='en')
             if data:

freqtrade/rpc/fiat_convert.py

@@ -39,7 +39,7 @@ class CryptoToFiatConverter(LoggingMixin):
     This object is also a Singleton
     """
     __instance = None
-    _coingekko: CoinGeckoAPI = None
+    _coingecko: CoinGeckoAPI = None
     _coinlistings: List[Dict] = []
     _backoff: float = 0.0
 
@@ -52,9 +52,9 @@ class CryptoToFiatConverter(LoggingMixin):
             try:
                 # Limit retires to 1 (0 and 1)
                 # otherwise we risk bot impact if coingecko is down.
-                CryptoToFiatConverter._coingekko = CoinGeckoAPI(retries=1)
+                CryptoToFiatConverter._coingecko = CoinGeckoAPI(retries=1)
             except BaseException:
-                CryptoToFiatConverter._coingekko = None
+                CryptoToFiatConverter._coingecko = None
         return CryptoToFiatConverter.__instance
 
     def __init__(self) -> None:
@@ -67,7 +67,7 @@ class CryptoToFiatConverter(LoggingMixin):
     def _load_cryptomap(self) -> None:
         try:
             # Use list-comprehension to ensure we get a list.
-            self._coinlistings = [x for x in self._coingekko.get_coins_list()]
+            self._coinlistings = [x for x in self._coingecko.get_coins_list()]
         except RequestException as request_exception:
             if "429" in str(request_exception):
                 logger.warning(
@@ -84,7 +84,7 @@ class CryptoToFiatConverter(LoggingMixin):
             logger.error(
                 f"Could not load FIAT Cryptocurrency map for the following problem: {exception}")
 
-    def _get_gekko_id(self, crypto_symbol):
+    def _get_gecko_id(self, crypto_symbol):
         if not self._coinlistings:
             if self._backoff <= datetime.now().timestamp():
                 self._load_cryptomap()
@@ -180,9 +180,9 @@ class CryptoToFiatConverter(LoggingMixin):
         if crypto_symbol == fiat_symbol:
             return 1.0
 
-        _gekko_id = self._get_gekko_id(crypto_symbol)
+        _gecko_id = self._get_gecko_id(crypto_symbol)
 
-        if not _gekko_id:
+        if not _gecko_id:
             # return 0 for unsupported stake currencies (fiat-convert should not break the bot)
             self.log_once(
                 f"unsupported crypto-symbol {crypto_symbol.upper()} - returning 0.0",
@@ -191,10 +191,10 @@ class CryptoToFiatConverter(LoggingMixin):
 
         try:
             return float(
-                self._coingekko.get_price(
-                    ids=_gekko_id,
+                self._coingecko.get_price(
+                    ids=_gecko_id,
                     vs_currencies=fiat_symbol
-                )[_gekko_id][fiat_symbol]
+                )[_gecko_id][fiat_symbol]
             )
         except Exception as exception:
             logger.error("Error in _find_price: %s", exception)

tests/conftest.py

@@ -490,10 +490,10 @@ def user_dir(mocker, tmp_path) -> Path:
 
 
 @pytest.fixture(autouse=True)
-def patch_coingekko(mocker) -> None:
+def patch_coingecko(mocker) -> None:
     """
-    Mocker to coingekko to speed up tests
-    :param mocker: mocker to patch coingekko class
+    Mocker to coingecko to speed up tests
+    :param mocker: mocker to patch coingecko class
     :return: None
     """
 

tests/freqtradebot/test_freqtradebot.py

@@ -4558,6 +4558,67 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor
     assert trade.exit_reason == ExitType.SOLD_ON_EXCHANGE.value
 
 
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize("is_short", [False, True])
+@pytest.mark.parametrize("factor,adjusts", [
+    (0.99, True),
+    (0.97, False),
+])
+def test_handle_onexchange_order_changed_amount(
+    mocker, default_conf_usdt, limit_order, is_short, caplog,
+    factor, adjusts,
+):
+    default_conf_usdt['dry_run'] = False
+    freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
+    mock_uts = mocker.spy(freqtrade, 'update_trade_state')
+
+    entry_order = limit_order[entry_side(is_short)]
+    mock_fo = mocker.patch(f'{EXMS}.fetch_orders', return_value=[
+        entry_order,
+    ])
+
+    trade = Trade(
+        pair='ETH/USDT',
+        fee_open=0.001,
+        base_currency='ETH',
+        fee_close=0.001,
+        open_rate=entry_order['price'],
+        open_date=dt_now(),
+        stake_amount=entry_order['cost'],
+        amount=entry_order['amount'],
+        exchange="binance",
+        is_short=is_short,
+        leverage=1,
+    )
+    freqtrade.wallets = MagicMock()
+    freqtrade.wallets.get_total = MagicMock(return_value=entry_order['amount'] * factor)
+
+    trade.orders.append(Order.parse_from_ccxt_object(
+        entry_order, 'ADA/USDT', entry_side(is_short))
+    )
+    Trade.session.add(trade)
+
+    # assert trade.amount > entry_order['amount']
+
+    freqtrade.handle_onexchange_order(trade)
+    assert mock_uts.call_count == 1
+    assert mock_fo.call_count == 1
+
+    trade = Trade.session.scalars(select(Trade)).first()
+
+    assert log_has_re(r'.*has a total of .* but the Wallet shows.*', caplog)
+    if adjusts:
+        # Trade amount is updated
+        assert trade.amount == entry_order['amount'] * factor
+        assert log_has_re(r'.*Adjusting trade amount to.*', caplog)
+    else:
+        assert log_has_re(r'.*Refusing to adjust as the difference.*', caplog)
+        assert trade.amount == entry_order['amount']
+
+    assert len(trade.orders) == 1
+    assert trade.is_open is True
+
+
 @pytest.mark.usefixtures("init_persistence")
 @pytest.mark.parametrize("is_short", [False, True])
 def test_handle_onexchange_order_exit(mocker, default_conf_usdt, limit_order, is_short, caplog):

tests/optimize/test_hyperopt_tools.py

@@ -172,8 +172,8 @@ def test__pprint_dict():
 }"""
 
 
-def test_get_strategy_filename(default_conf):
-
+def test_get_strategy_filename(default_conf, tmp_path):
+    default_conf['user_data_dir'] = tmp_path
     x = HyperoptTools.get_strategy_filename(default_conf, 'StrategyTestV3')
     assert isinstance(x, Path)
     assert x == Path(__file__).parents[1] / 'strategy/strats/strategy_test_v3.py'
@@ -233,6 +233,7 @@ def test_export_params(tmp_path):
 
 def test_try_export_params(default_conf, tmp_path, caplog, mocker):
     default_conf['disableparamexport'] = False
+    default_conf['user_data_dir'] = tmp_path
     export_mock = mocker.patch("freqtrade.optimize.hyperopt_tools.HyperoptTools.export_params")
 
     filename = tmp_path / f"{CURRENT_TEST_STRATEGY}.json"

tests/optimize/test_lookahead_analysis.py

@@ -14,7 +14,8 @@ from tests.conftest import EXMS, get_args, log_has_re, patch_exchange
 
 
 @pytest.fixture
-def lookahead_conf(default_conf_usdt):
+def lookahead_conf(default_conf_usdt, tmp_path):
+    default_conf_usdt['user_data_dir'] = tmp_path
     default_conf_usdt['minimum_trade_amount'] = 10
     default_conf_usdt['targeted_trade_amount'] = 20
     default_conf_usdt['timerange'] = '20220101-20220501'

tests/optimize/test_recursive_analysis.py

@@ -14,7 +14,8 @@ from tests.conftest import get_args, log_has_re, patch_exchange
 
 
 @pytest.fixture
-def recursive_conf(default_conf_usdt):
+def recursive_conf(default_conf_usdt, tmp_path):
+    default_conf_usdt['user_data_dir'] = tmp_path
     default_conf_usdt['timerange'] = '20220101-20220501'
 
     default_conf_usdt['strategy_path'] = str(

tests/rpc/test_fiat_convert.py

@@ -90,7 +90,7 @@ def test_loadcryptomap(mocker):
     fiat_convert = CryptoToFiatConverter()
     assert len(fiat_convert._coinlistings) == 2
 
-    assert fiat_convert._get_gekko_id("btc") == "bitcoin"
+    assert fiat_convert._get_gecko_id("btc") == "bitcoin"
 
 
 def test_fiat_init_network_exception(mocker):
@@ -109,16 +109,16 @@ def test_fiat_init_network_exception(mocker):
 
 
 def test_fiat_convert_without_network(mocker):
-    # Because CryptoToFiatConverter is a Singleton we reset the value of _coingekko
+    # Because CryptoToFiatConverter is a Singleton we reset the value of _coingecko
 
     fiat_convert = CryptoToFiatConverter()
 
-    cmc_temp = CryptoToFiatConverter._coingekko
-    CryptoToFiatConverter._coingekko = None
+    cmc_temp = CryptoToFiatConverter._coingecko
+    CryptoToFiatConverter._coingecko = None
 
-    assert fiat_convert._coingekko is None
+    assert fiat_convert._coingecko is None
     assert fiat_convert._find_price(crypto_symbol='btc', fiat_symbol='usd') == 0.0
-    CryptoToFiatConverter._coingekko = cmc_temp
+    CryptoToFiatConverter._coingecko = cmc_temp
 
 
 def test_fiat_too_many_requests_response(mocker, caplog):
@@ -152,9 +152,9 @@ def test_fiat_multiple_coins(mocker, caplog):
         {'id': 'ethereum-wormhole', 'symbol': 'eth', 'name': 'Ethereum Wormhole'},
     ]
 
-    assert fiat_convert._get_gekko_id('btc') == 'bitcoin'
-    assert fiat_convert._get_gekko_id('hnt') is None
-    assert fiat_convert._get_gekko_id('eth') == 'ethereum'
+    assert fiat_convert._get_gecko_id('btc') == 'bitcoin'
+    assert fiat_convert._get_gecko_id('hnt') is None
+    assert fiat_convert._get_gecko_id('eth') == 'ethereum'
 
     assert log_has('Found multiple mappings in CoinGecko for hnt.', caplog)
 

tests/rpc/test_rpc_apiserver.py

@@ -1577,8 +1577,10 @@ def test_api_pair_candles(botclient, ohlcv_history):
              ])
 
 
-def test_api_pair_history(botclient, mocker):
+def test_api_pair_history(botclient, tmp_path, mocker):
     _ftbot, client = botclient
+    _ftbot.config['user_data_dir'] = tmp_path
+
     timeframe = '5m'
     lfm = mocker.patch('freqtrade.strategy.interface.IStrategy.load_freqAI_model')
     # No pair
@@ -1648,8 +1650,9 @@ def test_api_pair_history(botclient, mocker):
     assert rc.json()['detail'] == ("No data for UNITTEST/BTC, 5m in 20200111-20200112 found.")
 
 
-def test_api_plot_config(botclient, mocker):
+def test_api_plot_config(botclient, mocker, tmp_path):
     ftbot, client = botclient
+    ftbot.config['user_data_dir'] = tmp_path
 
     rc = client_get(client, f"{BASE_URI}/plot_config")
     assert_response(rc)
@@ -1717,8 +1720,9 @@ def test_api_strategies(botclient, tmp_path):
     ]}
 
 
-def test_api_strategy(botclient):
+def test_api_strategy(botclient, tmp_path):
     _ftbot, client = botclient
+    _ftbot.config['user_data_dir'] = tmp_path
 
     rc = client_get(client, f"{BASE_URI}/strategy/{CURRENT_TEST_STRATEGY}")
 

tests/strategy/test_strategy_loading.py

@@ -78,7 +78,9 @@ def test_load_strategy_base64(dataframe_1m, caplog, default_conf):
                       r".*(/|\\).*(/|\\)SampleStrategy\.py'\.\.\.", caplog)
 
 
-def test_load_strategy_invalid_directory(caplog, default_conf):
+def test_load_strategy_invalid_directory(caplog, default_conf, tmp_path):
+    default_conf['user_data_dir'] = tmp_path
+
     extra_dir = Path.cwd() / 'some/path'
     with pytest.raises(OperationalException, match=r"Impossible to load Strategy.*"):
         StrategyResolver._load_strategy('StrategyTestV333', config=default_conf,
@@ -87,7 +89,8 @@ def test_load_strategy_invalid_directory(caplog, default_conf):
     assert log_has_re(r'Path .*' + r'some.*path.*' + r'.* does not exist', caplog)
 
 
-def test_load_not_found_strategy(default_conf):
+def test_load_not_found_strategy(default_conf, tmp_path):
+    default_conf['user_data_dir'] = tmp_path
     default_conf['strategy'] = 'NotFoundStrategy'
     with pytest.raises(OperationalException,
                        match=r"Impossible to load Strategy 'NotFoundStrategy'. "