From 3d6079ae19108f09205001d485974b837e4329a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Feb 2024 17:47:43 +0100 Subject: [PATCH 01/54] Add debug output showing the pair to be converted part of #9811 --- freqtrade/data/converter/trade_converter_kraken.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/data/converter/trade_converter_kraken.py b/freqtrade/data/converter/trade_converter_kraken.py index 614d97b2a..c9848c096 100644 --- a/freqtrade/data/converter/trade_converter_kraken.py +++ b/freqtrade/data/converter/trade_converter_kraken.py @@ -48,6 +48,7 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str): logger.info(f"Converting pairs: {', '.join(m[0] for m in markets)}.") for pair, name in markets: + logger.debug(f"Converting pair {pair}, files */{name}.csv") dfs = [] # Load and combine all csv files for this pair for f in tradesdir.rglob(f"{name}.csv"): From 280737447cdac4806161fe6e0f72a3b808d68b5c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Feb 2024 07:26:23 +0100 Subject: [PATCH 02/54] Don't load empty dataframes - skip these closes #9811 --- freqtrade/data/converter/trade_converter_kraken.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/converter/trade_converter_kraken.py b/freqtrade/data/converter/trade_converter_kraken.py index c9848c096..b0fa11c25 100644 --- a/freqtrade/data/converter/trade_converter_kraken.py +++ b/freqtrade/data/converter/trade_converter_kraken.py @@ -53,7 +53,8 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str): # Load and combine all csv files for this pair for f in tradesdir.rglob(f"{name}.csv"): df = pd.read_csv(f, names=KRAKEN_CSV_TRADE_COLUMNS) - dfs.append(df) + if not df.empty: + dfs.append(df) # Load existing trades data if not dfs: From 57fd0e379abe1161390668c2dd6cd4a8c3925ce1 Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Thu, 15 Feb 2024 15:57:49 +0000 Subject: [PATCH 03/54] Clarify processing_mode for RemotePairlist No default value is specified in the docs for the processing_mode, making it unclear that the default behaviour is to filter out pairs, rather than append. --- docs/includes/pairlists.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 9781edf10..5a6a2560b 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -201,7 +201,7 @@ The RemotePairList is defined in the pairlists section of the configuration sett The optional `mode` option specifies if the pairlist should be used as a `blacklist` or as a `whitelist`. The default value is "whitelist". -The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append". +The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append". The default value is "filter". In "filter" mode, the retrieved pairlist is used as a filter. Only the pairs present in both the original pairlist and the retrieved pairlist are included in the final pairlist. Other pairs are filtered out. From 86da9cb659d126a3a46a4eec2c3a25ab09761a9f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Feb 2024 19:27:15 +0100 Subject: [PATCH 04/54] Simplify some pairlist conditions --- freqtrade/plugins/pairlist/VolatilityFilter.py | 3 +-- freqtrade/plugins/pairlist/rangestabilityfilter.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 800bf3664..794df5449 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -125,8 +125,7 @@ class VolatilityFilter(IPairList): :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache - cached_res = self._pair_cache.get(pair, None) - if cached_res is not None: + if (cached_res := self._pair_cache.get(pair, None)) is not None: return cached_res result = False diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index f4625f572..e04772e9c 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -123,8 +123,7 @@ class RangeStabilityFilter(IPairList): :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache - cached_res = self._pair_cache.get(pair, None) - if cached_res is not None: + if (cached_res := self._pair_cache.get(pair, None)) is not None: return cached_res result = True From d01e9cf2990e8bf21199b3aed465387be2da8d3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Feb 2024 19:49:47 +0100 Subject: [PATCH 05/54] Improve log message --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ff04037da..2032e437d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -702,7 +702,7 @@ class FreqtradeBot(LoggingMixin): delta = f"Delta: {bids_ask_delta}" logger.info( - f"{bids}, {asks}, {delta}, Direction: {side.value}" + f"{bids}, {asks}, {delta}, Direction: {side.value} " f"Bid Price: {order_book['bids'][0][0]}, Ask Price: {order_book['asks'][0][0]}, " f"Immediate Bid Quantity: {order_book['bids'][0][1]}, " f"Immediate Ask Quantity: {order_book['asks'][0][1]}." From 6c9b9e91e8f47b94559c5ab9862b243c8165437e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Feb 2024 06:39:07 +0100 Subject: [PATCH 06/54] enhance volumpairlist range test --- tests/plugins/test_pairlist.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 09dcd0af3..55d65d3c7 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -626,8 +626,9 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t # "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}], # "BTC", "ftx", ['HOT/BTC', 'LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'XRP/BTC']), ]) -def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history, - pairlists, base_currency, exchange, volumefilter_result) -> None: +def test_VolumePairList_range( + mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history, + pairlists, base_currency, exchange, volumefilter_result, time_machine) -> None: whitelist_conf['pairlists'] = pairlists whitelist_conf['stake_currency'] = base_currency whitelist_conf['exchange']['name'] = exchange @@ -686,23 +687,35 @@ def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, get_tickers=tickers, markets=PropertyMock(return_value=shitcoinmarkets) ) - + start_dt = dt_now() + time_machine.move_to(start_dt) # remove ohlcv when looback_timeframe != 1d # to enforce fallback to ticker data if 'lookback_timeframe' in pairlists[0]: if pairlists[0]['lookback_timeframe'] != '1d': ohlcv_data = [] - mocker.patch.multiple( - EXMS, - refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), - ) + ohclv_mock = mocker.patch(f"{EXMS}.refresh_latest_ohlcv", return_value=ohlcv_data) freqtrade.pairlists.refresh_pairlist() whitelist = freqtrade.pairlists.whitelist + assert ohclv_mock.call_count == 1 assert isinstance(whitelist, list) assert whitelist == volumefilter_result + # Test caching + ohclv_mock.reset_mock() + freqtrade.pairlists.refresh_pairlist() + assert ohclv_mock.call_count == 0 + whitelist = freqtrade.pairlists.whitelist + assert whitelist == volumefilter_result + + time_machine.move_to(start_dt + timedelta(days=2)) + ohclv_mock.reset_mock() + freqtrade.pairlists.refresh_pairlist() + assert ohclv_mock.call_count == 1 + whitelist = freqtrade.pairlists.whitelist + assert whitelist == volumefilter_result def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: From 7f7e9ec8756b2bd2b555e4b14b4ba0c5af476baa Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Feb 2024 06:42:57 +0100 Subject: [PATCH 07/54] Add additional test case for VolumePairlist in range mode --- tests/plugins/test_pairlist.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 55d65d3c7..32f6abb51 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -621,6 +621,12 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", "lookback_timeframe": "1d", "lookback_period": 6, "refresh_period": 86400}], "BTC", "binance", ['LTC/BTC', 'XRP/BTC', 'ETH/BTC', 'HOT/BTC', 'NEO/BTC']), + # VolumePairlist in range mode as filter. + # TKN/BTC is removed because it doesn't have enough candles + ([{"method": "VolumePairList", "number_assets": 5}, + {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1d", "lookback_period": 2, "refresh_period": 86400}], + "BTC", "binance", ['LTC/BTC', 'XRP/BTC', 'ETH/BTC', 'TKN/BTC', 'HOT/BTC']), # ftx data is already in Quote currency, therefore won't require conversion # ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", # "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}], @@ -693,7 +699,7 @@ def test_VolumePairList_range( # to enforce fallback to ticker data if 'lookback_timeframe' in pairlists[0]: if pairlists[0]['lookback_timeframe'] != '1d': - ohlcv_data = [] + ohlcv_data = {} ohclv_mock = mocker.patch(f"{EXMS}.refresh_latest_ohlcv", return_value=ohlcv_data) @@ -706,7 +712,8 @@ def test_VolumePairList_range( # Test caching ohclv_mock.reset_mock() freqtrade.pairlists.refresh_pairlist() - assert ohclv_mock.call_count == 0 + # in "filter" mode, caching is disabled. + assert ohclv_mock.call_count == (0 if len(pairlists) == 1 else 1) whitelist = freqtrade.pairlists.whitelist assert whitelist == volumefilter_result From a22181d721ea465ce27fca653cff56a00b66eae8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Feb 2024 18:16:55 +0100 Subject: [PATCH 08/54] Enable caching for "filter only" Volumepairlist --- freqtrade/plugins/pairlist/VolumePairList.py | 16 ++++++++++++---- tests/plugins/test_pairlist.py | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index b5525e950..671ba4db4 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -14,7 +14,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter -from freqtrade.util import dt_now, format_ms_time +from freqtrade.util import PeriodicCache, dt_now, format_ms_time logger = logging.getLogger(__name__) @@ -63,6 +63,7 @@ class VolumePairList(IPairList): # get timeframe in minutes and seconds self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe) _tf_in_sec = self._tf_in_min * 60 + self._candle_cache = PeriodicCache(maxsize=1000, ttl=_tf_in_sec) # wether to use range lookback or not self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0) @@ -230,11 +231,18 @@ class VolumePairList(IPairList): ] # Get all candles - candles = {} - if needed_pairs: + candles = { + c: self._candle_cache.get(c, None) for c in needed_pairs + if c in self._candle_cache + } + pairs_to_download = [p for p in needed_pairs if p not in candles] + if pairs_to_download: candles = self._exchange.refresh_latest_ohlcv( - needed_pairs, since_ms=since_ms, cache=False + pairs_to_download, since_ms=since_ms, cache=False ) + for c, val in candles.items(): + self._candle_cache[c] = val + for i, p in enumerate(filtered_tickers): contract_size = self._exchange.markets[p['symbol']].get('contractSize', 1.0) or 1.0 pair_candles = candles[ diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 32f6abb51..d125f8896 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -713,7 +713,7 @@ def test_VolumePairList_range( ohclv_mock.reset_mock() freqtrade.pairlists.refresh_pairlist() # in "filter" mode, caching is disabled. - assert ohclv_mock.call_count == (0 if len(pairlists) == 1 else 1) + assert ohclv_mock.call_count == 0 whitelist = freqtrade.pairlists.whitelist assert whitelist == volumefilter_result From c1d71848490a8530b4e5ef66f21a60baa5c2e6b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Feb 2024 19:20:08 +0100 Subject: [PATCH 09/54] Adjust for ccxt exception hierarchy change caused by https://github.com/ccxt/ccxt/pull/21035 --- freqtrade/exchange/exchange.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 85a77fe5e..4c142a517 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1242,7 +1242,7 @@ class Exchange: f'Insufficient funds to create {ordertype} {side} order on market {pair}. ' f'Tried to {side} amount {amount} at rate {limit_rate} with ' f'stop-price {stop_price_norm}. Message: {e}') from e - except (ccxt.InvalidOrder, ccxt.BadRequest) as e: + except (ccxt.InvalidOrder, ccxt.BadRequest, ccxt.OperationRejected) as e: # Errors: # `Order would trigger immediately.` raise InvalidOrderException( @@ -2685,7 +2685,7 @@ class Exchange: self._log_exchange_response('set_leverage', res) except ccxt.DDoSProtection as e: raise DDosProtection(e) from e - except (ccxt.BadRequest, ccxt.InsufficientFunds) as e: + except (ccxt.BadRequest, ccxt.OperationRejected, ccxt.InsufficientFunds) as e: if not accept_fail: raise TemporaryError( f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e @@ -2727,7 +2727,7 @@ class Exchange: self._log_exchange_response('set_margin_mode', res) except ccxt.DDoSProtection as e: raise DDosProtection(e) from e - except ccxt.BadRequest as e: + except (ccxt.BadRequest, ccxt.OperationRejected) as e: if not accept_fail: raise TemporaryError( f'Could not set margin mode due to {e.__class__.__name__}. Message: {e}') from e From c6d1c1a980ed9f587e16fa0b9df38d075b185545 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Feb 2024 19:50:56 +0100 Subject: [PATCH 10/54] Add dt_ts_none helper --- freqtrade/util/__init__.py | 5 +++-- freqtrade/util/datetime_helpers.py | 11 ++++++++++- tests/utils/test_datetime_helpers.py | 11 +++++++++-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 513406fd2..f7e63d9d3 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,6 +1,6 @@ from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, - dt_ts_def, dt_utc, format_date, format_ms_time, - shorten_date) + dt_ts_def, dt_ts_none, dt_utc, format_date, + format_ms_time, shorten_date) from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache @@ -14,6 +14,7 @@ __all__ = [ 'dt_now', 'dt_ts', 'dt_ts_def', + 'dt_ts_none', 'dt_utc', 'format_date', 'format_ms_time', diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index 102c83143..973a1c51b 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -31,12 +31,21 @@ def dt_ts(dt: Optional[datetime] = None) -> int: def dt_ts_def(dt: Optional[datetime], default: int = 0) -> int: """ Return dt in ms as a timestamp in UTC. - If dt is None, return the current datetime in UTC. + If dt is None, return the given default. """ if dt: return int(dt.timestamp() * 1000) return default +def dt_ts_none(dt: Optional[datetime]) -> Optional[int]: + """ + Return dt in ms as a timestamp in UTC. + If dt is None, return the given default. + """ + if dt: + return int(dt.timestamp() * 1000) + return None + def dt_floor_day(dt: datetime) -> datetime: """Return the floor of the day for the given datetime.""" diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index b70065645..6fbe75200 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -3,8 +3,8 @@ from datetime import datetime, timedelta, timezone import pytest import time_machine -from freqtrade.util import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, dt_ts_def, dt_utc, - format_date, format_ms_time, shorten_date) +from freqtrade.util import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, dt_ts_def, + dt_ts_none, dt_utc, format_date, format_ms_time, shorten_date) def test_dt_now(): @@ -29,6 +29,13 @@ def test_dt_ts_def(): assert dt_ts_def(datetime(2023, 5, 5, tzinfo=timezone.utc), 123) == 1683244800000 +def test_dt_ts_none(): + assert dt_ts_none(None) is None + assert dt_ts_none(None) is None + assert dt_ts_none(datetime(2023, 5, 5, tzinfo=timezone.utc)) == 1683244800000 + assert dt_ts_none(datetime(2023, 5, 5, tzinfo=timezone.utc)) == 1683244800000 + + def test_dt_utc(): assert dt_utc(2023, 5, 5) == datetime(2023, 5, 5, tzinfo=timezone.utc) assert dt_utc(2023, 5, 5, 0, 0, 0, 555500) == datetime(2023, 5, 5, 0, 0, 0, 555500, From 0f85ef09973a0f32df6488a3873f591888364d88 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Feb 2024 19:52:50 +0100 Subject: [PATCH 11/54] Simplify trade_model serializations --- freqtrade/persistence/trade_model.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index a90d9ab2d..84c11f02c 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -23,7 +23,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi from freqtrade.leverage import interest from freqtrade.misc import safe_value_fallback from freqtrade.persistence.base import ModelBase, SessionType -from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts +from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts, dt_ts_none logger = logging.getLogger(__name__) @@ -224,8 +224,7 @@ class Order(ModelBase): 'amount': self.safe_amount, 'safe_price': self.safe_price, 'ft_order_side': self.ft_order_side, - 'order_filled_timestamp': int(self.order_filled_date.replace( - tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None, + 'order_filled_timestamp': dt_ts_none(self.order_filled_utc), 'ft_is_entry': self.ft_order_side == entry_side, 'ft_order_tag': self.ft_order_tag, } @@ -625,15 +624,14 @@ class LocalTrade: 'fee_close_currency': self.fee_close_currency, 'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT), - 'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000), + 'open_timestamp': dt_ts_none(self.open_date_utc), 'open_rate': self.open_rate, 'open_rate_requested': self.open_rate_requested, 'open_trade_value': round(self.open_trade_value, 8), 'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT) if self.close_date else None), - 'close_timestamp': int(self.close_date.replace( - tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None, + 'close_timestamp': dt_ts_none(self.close_date_utc), 'realized_profit': self.realized_profit or 0.0, # Close-profit corresponds to relative realized_profit ratio 'realized_profit_ratio': self.close_profit or None, @@ -659,8 +657,7 @@ class LocalTrade: 'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None, 'stoploss_last_update': (self.stoploss_last_update_utc.strftime(DATETIME_PRINT_FORMAT) if self.stoploss_last_update_utc else None), - 'stoploss_last_update_timestamp': int(self.stoploss_last_update_utc.timestamp() * 1000 - ) if self.stoploss_last_update_utc else None, + 'stoploss_last_update_timestamp': dt_ts_none(self.stoploss_last_update_utc), 'initial_stop_loss_abs': self.initial_stop_loss, 'initial_stop_loss_ratio': (self.initial_stop_loss_pct if self.initial_stop_loss_pct else None), From fb54c9ffe4a4623e62ff36aa0917bd6782391940 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Feb 2024 20:19:08 +0100 Subject: [PATCH 12/54] Add open_fill_date stuff to json schema --- freqtrade/persistence/trade_model.py | 14 ++++++++++++++ freqtrade/rpc/api_server/api_schemas.py | 2 ++ 2 files changed, 16 insertions(+) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 84c11f02c..121a0bd8a 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -460,6 +460,17 @@ class LocalTrade: return self.open_date_utc return max([self.open_date_utc, dt_last_filled]) + @property + def date_entry_fill_utc(self) -> Optional[datetime]: + """ Date of the first filled order""" + orders = self.select_filled_orders(self.entry_side) + if ( + orders + and len((filled_date := [o.order_filled_utc for o in orders if o.order_filled_utc])) + ): + return min(filled_date) + return None + @property def open_date_utc(self): return self.open_date.replace(tzinfo=timezone.utc) @@ -625,6 +636,9 @@ class LocalTrade: 'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT), 'open_timestamp': dt_ts_none(self.open_date_utc), + 'open_fill_date': (self.date_entry_fill_utc.strftime(DATETIME_PRINT_FORMAT) + if self.date_entry_fill_utc else None), + 'open_fill_timestamp': dt_ts_none(self.date_entry_fill_utc), 'open_rate': self.open_rate, 'open_rate_requested': self.open_rate_requested, 'open_trade_value': round(self.open_trade_value, 8), diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 9919d1a05..3ea9ed4d0 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -288,6 +288,8 @@ class TradeSchema(BaseModel): open_date: str open_timestamp: int + open_fill_date: Optional[str] + open_fill_timestamp: Optional[int] open_rate: float open_rate_requested: Optional[float] = None open_trade_value: float From 1696aa391504cb2ade22a9124d17d7be27eea696 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Feb 2024 20:39:06 +0100 Subject: [PATCH 13/54] Adjust tests for new fields --- tests/persistence/test_persistence.py | 4 ++++ tests/persistence/test_trade_fromjson.py | 2 +- tests/rpc/test_rpc.py | 2 ++ tests/rpc/test_rpc_apiserver.py | 4 ++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 95db7bc0f..0e0e70ee8 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -1400,6 +1400,8 @@ def test_to_json(fee): 'is_open': None, 'open_date': trade.open_date.strftime(DATETIME_PRINT_FORMAT), 'open_timestamp': int(trade.open_date.timestamp() * 1000), + 'open_fill_date': None, + 'open_fill_timestamp': None, 'close_date': None, 'close_timestamp': None, 'open_rate': 0.123, @@ -1486,6 +1488,8 @@ def test_to_json(fee): 'quote_currency': 'BTC', 'open_date': trade.open_date.strftime(DATETIME_PRINT_FORMAT), 'open_timestamp': int(trade.open_date.timestamp() * 1000), + 'open_fill_date': None, + 'open_fill_timestamp': None, 'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT), 'close_timestamp': int(trade.close_date.timestamp() * 1000), 'open_rate': 0.123, diff --git a/tests/persistence/test_trade_fromjson.py b/tests/persistence/test_trade_fromjson.py index 302a81c54..988f7ed5b 100644 --- a/tests/persistence/test_trade_fromjson.py +++ b/tests/persistence/test_trade_fromjson.py @@ -223,7 +223,7 @@ def test_trade_serialize_load_back(fee): 'realized_profit_ratio', 'close_profit_pct', 'trade_duration_s', 'trade_duration', 'profit_ratio', 'profit_pct', 'profit_abs', 'stop_loss_abs', - 'initial_stop_loss_abs', + 'initial_stop_loss_abs', 'open_fill_date', 'open_fill_timestamp', 'orders', ] failed = [] diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 1f51b30df..85b105892 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -25,6 +25,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'quote_currency': 'BTC', 'open_date': ANY, 'open_timestamp': ANY, + 'open_fill_date': ANY, + 'open_fill_timestamp': ANY, 'is_open': ANY, 'fee_open': ANY, 'fee_open_cost': ANY, diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index bba18bcd3..e441b127b 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1165,6 +1165,8 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, 'current_rate': current_rate, 'open_date': ANY, 'open_timestamp': ANY, + 'open_fill_date': ANY, + 'open_fill_timestamp': ANY, 'open_rate': 0.123, 'pair': 'ETH/BTC', 'base_currency': 'ETH', @@ -1368,6 +1370,8 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): 'close_rate': 0.265441, 'open_date': ANY, 'open_timestamp': ANY, + 'open_fill_date': ANY, + 'open_fill_timestamp': ANY, 'open_rate': 0.245441, 'pair': 'ETH/BTC', 'base_currency': 'ETH', From fd48991fb073d5362a5c458e01f2e53d17847967 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Feb 2024 20:44:16 +0100 Subject: [PATCH 14/54] Fix duplicate parentheses --- freqtrade/persistence/trade_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 121a0bd8a..b1330b83c 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -466,7 +466,7 @@ class LocalTrade: orders = self.select_filled_orders(self.entry_side) if ( orders - and len((filled_date := [o.order_filled_utc for o in orders if o.order_filled_utc])) + and len(filled_date := [o.order_filled_utc for o in orders if o.order_filled_utc]) ): return min(filled_date) return None From 60b12c1d9ecd19e8ed7ae6e559021c810c068c8c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Feb 2024 20:45:16 +0100 Subject: [PATCH 15/54] Double newlines between functions ... --- freqtrade/util/datetime_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index 973a1c51b..66b738e8d 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -37,6 +37,7 @@ def dt_ts_def(dt: Optional[datetime], default: int = 0) -> int: return int(dt.timestamp() * 1000) return default + def dt_ts_none(dt: Optional[datetime]) -> Optional[int]: """ Return dt in ms as a timestamp in UTC. From a0b7df70d694a978dd7433244569883db8358e62 Mon Sep 17 00:00:00 2001 From: CaffeinatedTech Date: Fri, 16 Feb 2024 13:36:16 +1000 Subject: [PATCH 16/54] Added escaping to enter and exit tags on telegram performance messages. --- freqtrade/rpc/telegram.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index f9a0635f0..f28b26766 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -4,6 +4,7 @@ This module manage Telegram communication """ import asyncio +import html import json import logging import re @@ -1378,7 +1379,7 @@ class Telegram(RPCHandler): output = "Entry Tag Performance:\n" for i, trade in enumerate(trades): stat_line = ( - f"{i + 1}.\t {trade['enter_tag']}\t" + f"{i + 1}.\t {html.escape(trade['enter_tag'])}\t" f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") @@ -1410,7 +1411,7 @@ class Telegram(RPCHandler): output = "Exit Reason Performance:\n" for i, trade in enumerate(trades): stat_line = ( - f"{i + 1}.\t {trade['exit_reason']}\t" + f"{i + 1}.\t {html.escape(trade['exit_reason'])}\t" f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") @@ -1442,7 +1443,7 @@ class Telegram(RPCHandler): output = "Mix Tag Performance:\n" for i, trade in enumerate(trades): stat_line = ( - f"{i + 1}.\t {trade['mix_tag']}\t" + f"{i + 1}.\t {html.escape(trade['mix_tag'])}\t" f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") From c0da1b6922891afc66b3071237ce3220f1421cdd Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Feb 2024 20:04:49 +0100 Subject: [PATCH 17/54] Fix edge-case when calculating cagr edge-case with leveraged trades - yielding a negative final balance. closes #9820 --- freqtrade/data/metrics.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py index 7b45342bb..b37e0bb19 100644 --- a/freqtrade/data/metrics.py +++ b/freqtrade/data/metrics.py @@ -191,6 +191,9 @@ def calculate_cagr(days_passed: int, starting_balance: float, final_balance: flo :param final_balance: Final balance to calculate CAGR against :return: CAGR """ + if final_balance < 0: + # With leveraged trades, final_balance can become negative. + return 0 return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1 From 4761bf242750e032ca427d0ad3f688cc77b4e23d Mon Sep 17 00:00:00 2001 From: CaffeinatedTech Date: Sat, 17 Feb 2024 09:12:49 +1000 Subject: [PATCH 18/54] Change enter_tag, exit_reason, mix_tag performance messages from HTML to Markdown to fix some string encoding issues. --- freqtrade/rpc/telegram.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index f28b26766..2983eea38 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -4,7 +4,6 @@ This module manage Telegram communication """ import asyncio -import html import json import logging import re @@ -1376,21 +1375,21 @@ class Telegram(RPCHandler): pair = context.args[0] trades = self._rpc._rpc_enter_tag_performance(pair) - output = "Entry Tag Performance:\n" + output = "*Entry Tag Performance:*\n" for i, trade in enumerate(trades): stat_line = ( - f"{i + 1}.\t {html.escape(trade['enter_tag'])}\t" + f"{i + 1}.\t `{trade['enter_tag']}\t" f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " - f"({trade['count']})\n") + f"({trade['count']})`\n") if len(output + stat_line) >= MAX_MESSAGE_LENGTH: - await self._send_msg(output, parse_mode=ParseMode.HTML) + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2) output = stat_line else: output += stat_line - await self._send_msg(output, parse_mode=ParseMode.HTML, + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2, reload_able=True, callback_path="update_enter_tag_performance", query=update.callback_query) @@ -1408,21 +1407,21 @@ class Telegram(RPCHandler): pair = context.args[0] trades = self._rpc._rpc_exit_reason_performance(pair) - output = "Exit Reason Performance:\n" + output = "*Exit Reason Performance:*\n" for i, trade in enumerate(trades): stat_line = ( - f"{i + 1}.\t {html.escape(trade['exit_reason'])}\t" + f"{i + 1}\.\t `{html.escape(trade['exit_reason'])}\t" f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " - f"({trade['count']})\n") + f"({trade['count']})`\n") if len(output + stat_line) >= MAX_MESSAGE_LENGTH: - await self._send_msg(output, parse_mode=ParseMode.HTML) + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2) output = stat_line else: output += stat_line - await self._send_msg(output, parse_mode=ParseMode.HTML, + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2, reload_able=True, callback_path="update_exit_reason_performance", query=update.callback_query) @@ -1440,21 +1439,21 @@ class Telegram(RPCHandler): pair = context.args[0] trades = self._rpc._rpc_mix_tag_performance(pair) - output = "Mix Tag Performance:\n" + output = "*Mix Tag Performance:*\n" for i, trade in enumerate(trades): stat_line = ( - f"{i + 1}.\t {html.escape(trade['mix_tag'])}\t" + f"{i + 1}\.\t `{trade['mix_tag']}\t" f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " - f"({trade['count']})\n") + f"({trade['count']})`\n") if len(output + stat_line) >= MAX_MESSAGE_LENGTH: - await self._send_msg(output, parse_mode=ParseMode.HTML) + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2) output = stat_line else: output += stat_line - await self._send_msg(output, parse_mode=ParseMode.HTML, + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2, reload_able=True, callback_path="update_mix_tag_performance", query=update.callback_query) From 3f3760c0ae7fc3fee5147737907b35d92375dd6b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Feb 2024 12:02:26 +0100 Subject: [PATCH 19/54] Use Markdown V1 - update tests --- freqtrade/rpc/telegram.py | 16 ++++++++-------- tests/rpc/test_rpc_telegram.py | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 2983eea38..fcc61b5e4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1384,12 +1384,12 @@ class Telegram(RPCHandler): f"({trade['count']})`\n") if len(output + stat_line) >= MAX_MESSAGE_LENGTH: - await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2) + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN) output = stat_line else: output += stat_line - await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2, + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN, reload_able=True, callback_path="update_enter_tag_performance", query=update.callback_query) @@ -1410,18 +1410,18 @@ class Telegram(RPCHandler): output = "*Exit Reason Performance:*\n" for i, trade in enumerate(trades): stat_line = ( - f"{i + 1}\.\t `{html.escape(trade['exit_reason'])}\t" + f"{i + 1}.\t `{trade['exit_reason']}\t" f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " f"({trade['count']})`\n") if len(output + stat_line) >= MAX_MESSAGE_LENGTH: - await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2) + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN) output = stat_line else: output += stat_line - await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2, + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN, reload_able=True, callback_path="update_exit_reason_performance", query=update.callback_query) @@ -1442,18 +1442,18 @@ class Telegram(RPCHandler): output = "*Mix Tag Performance:*\n" for i, trade in enumerate(trades): stat_line = ( - f"{i + 1}\.\t `{trade['mix_tag']}\t" + f"{i + 1}.\t `{trade['mix_tag']}\t" f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " f"({trade['count']})`\n") if len(output + stat_line) >= MAX_MESSAGE_LENGTH: - await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2) + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN) output = stat_line else: output += stat_line - await self._send_msg(output, parse_mode=ParseMode.MARKDOWN_V2, + await self._send_msg(output, parse_mode=ParseMode.MARKDOWN, reload_able=True, callback_path="update_mix_tag_performance", query=update.callback_query) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 3c683d7b3..7b1347fd6 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1507,7 +1507,7 @@ async def test_telegram_entry_tag_performance_handle( await telegram._enter_tag_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Entry Tag Performance' in msg_mock.call_args_list[0][0][0] - assert 'TEST1\t3.987 USDT (5.00%) (1)' in msg_mock.call_args_list[0][0][0] + assert '`TEST1\t3.987 USDT (5.00%) (1)`' in msg_mock.call_args_list[0][0][0] context.args = ['XRP/USDT'] await telegram._enter_tag_performance(update=update, context=context) @@ -1538,7 +1538,7 @@ async def test_telegram_exit_reason_performance_handle( await telegram._exit_reason_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Exit Reason Performance' in msg_mock.call_args_list[0][0][0] - assert 'roi\t2.842 USDT (10.00%) (1)' in msg_mock.call_args_list[0][0][0] + assert '`roi\t2.842 USDT (10.00%) (1)`' in msg_mock.call_args_list[0][0][0] context.args = ['XRP/USDT'] await telegram._exit_reason_performance(update=update, context=context) @@ -1570,7 +1570,7 @@ async def test_telegram_mix_tag_performance_handle(default_conf_usdt, update, ti await telegram._mix_tag_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Mix Tag Performance' in msg_mock.call_args_list[0][0][0] - assert ('TEST3 roi\t2.842 USDT (10.00%) (1)' + assert ('`TEST3 roi\t2.842 USDT (10.00%) (1)`' in msg_mock.call_args_list[0][0][0]) context.args = ['XRP/USDT'] From 8033faa2f29fc58905fd7f0794d0100ad0928bc9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Feb 2024 15:14:11 +0100 Subject: [PATCH 20/54] Update pairlist cache behavior in VolumePairList --- docs/includes/pairlists.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 5a6a2560b..d1dd2cda7 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -68,7 +68,7 @@ When used in the leading position of the chain of Pairlist Handlers, the `pair_w The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists. -Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data. +Filtering instances (not the first position in the list) will not apply any cache (beyond caching candles for the duration of the candle in advanced mode) and will always use up-to-date data. `VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library: From bcfe7ef547871b0f1f7e1984e23935aa511ca3ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Feb 2024 16:17:32 +0100 Subject: [PATCH 21/54] Refactor ohlcv caching to exchange class --- freqtrade/exchange/exchange.py | 36 ++++++++++++++++++++ freqtrade/plugins/pairlist/VolumePairList.py | 16 ++------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 85a77fe5e..6dbb38bd8 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -6,6 +6,7 @@ import asyncio import inspect import logging import signal +from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta, timezone from math import floor @@ -43,6 +44,7 @@ from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_ from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.util import dt_from_ts, dt_now from freqtrade.util.datetime_helpers import dt_humanize, dt_ts +from freqtrade.util.periodic_cache import PeriodicCache logger = logging.getLogger(__name__) @@ -131,6 +133,7 @@ class Exchange: # Holds candles self._klines: Dict[PairWithTimeframe, DataFrame] = {} + self._expiring_candle_cache: Dict[str, PeriodicCache] = {} # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} @@ -2124,6 +2127,39 @@ class Exchange: return results_df + def refresh_ohlcv_with_cache( + self, + pairs: List[PairWithTimeframe], + since_ms: int + ) -> Dict[PairWithTimeframe, DataFrame]: + """ + Refresh ohlcv data for all pairs in needed_pairs if necessary. + Caches data with expiring per timeframe. + Should only be used for pairlists which need "on time" expirarion, and no longer cache. + """ + + timeframes = [p[1] for p in pairs] + for timeframe in timeframes: + if timeframe not in self._expiring_candle_cache: + timeframe_in_sec = timeframe_to_seconds(timeframe) + # Initialise cache + self._expiring_candle_cache[timeframe] = PeriodicCache(ttl=timeframe_in_sec, + maxsize=1000) + + # Get candles from cache + candles = { + c: self._expiring_candle_cache[c[1]].get(c, None) for c in pairs + if c in self._expiring_candle_cache[c[1]] + } + pairs_to_download = [p for p in pairs if p not in candles] + if pairs_to_download: + candles = self.refresh_latest_ohlcv( + pairs_to_download, since_ms=since_ms, cache=False + ) + for c, val in candles.items(): + self._expiring_candle_cache[c[1]][c] = val + return candles + def _now_is_time_to_refresh(self, pair: str, timeframe: str, candle_type: CandleType) -> bool: # Timeframe in seconds interval_in_sec = timeframe_to_seconds(timeframe) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 671ba4db4..f4d08e800 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -14,7 +14,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter -from freqtrade.util import PeriodicCache, dt_now, format_ms_time +from freqtrade.util import dt_now, format_ms_time logger = logging.getLogger(__name__) @@ -63,7 +63,6 @@ class VolumePairList(IPairList): # get timeframe in minutes and seconds self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe) _tf_in_sec = self._tf_in_min * 60 - self._candle_cache = PeriodicCache(maxsize=1000, ttl=_tf_in_sec) # wether to use range lookback or not self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0) @@ -230,18 +229,7 @@ class VolumePairList(IPairList): if p not in self._pair_cache ] - # Get all candles - candles = { - c: self._candle_cache.get(c, None) for c in needed_pairs - if c in self._candle_cache - } - pairs_to_download = [p for p in needed_pairs if p not in candles] - if pairs_to_download: - candles = self._exchange.refresh_latest_ohlcv( - pairs_to_download, since_ms=since_ms, cache=False - ) - for c, val in candles.items(): - self._candle_cache[c] = val + candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms) for i, p in enumerate(filtered_tickers): contract_size = self._exchange.markets[p['symbol']].get('contractSize', 1.0) or 1.0 From 7b36a0fc4220ae3d7141228ad9e4a23a29a07e7d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Feb 2024 16:26:53 +0100 Subject: [PATCH 22/54] Add explicit test for ohlcv_with_cache --- tests/exchange/test_exchange.py | 60 +++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index fc199a7f5..ef41a6eb0 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2303,6 +2303,66 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach assert res[pair2].at[0, 'open'] +def test_refresh_ohlcv_with_cache(mocker, default_conf, time_machine) -> None: + start = datetime(2021, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc) + ohlcv = generate_test_data_raw('1h', 100, start.strftime('%Y-%m-%d')) + time_machine.move_to(start, tick=False) + pairs = [ + ('ETH/BTC', '1d', CandleType.SPOT), + ('TKN/BTC', '1d', CandleType.SPOT), + ('LTC/BTC', '1d', CandleType.SPOT), + ('LTC/BTC', '5m', CandleType.SPOT), + ('LTC/BTC', '1h', CandleType.SPOT), + ] + + ohlcv_data = { + p: ohlcv for p in pairs + } + ohlcv_mock = mocker.patch(f"{EXMS}.refresh_latest_ohlcv", return_value=ohlcv_data) + mocker.patch(f"{EXMS}.ohlcv_candle_limit", return_value=100) + exchange = get_patched_exchange(mocker, default_conf) + + assert len(exchange._expiring_candle_cache) == 0 + + res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp()) + assert ohlcv_mock.call_count == 1 + assert ohlcv_mock.call_args_list[0][0][0] == pairs + assert len(ohlcv_mock.call_args_list[0][0][0]) == 5 + + assert len(res) == 5 + # length of 3 - as we have 3 different timeframes + assert len(exchange._expiring_candle_cache) == 3 + + ohlcv_mock.reset_mock() + res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp()) + assert ohlcv_mock.call_count == 0 + + # Expire 5m cache + time_machine.move_to(start + timedelta(minutes=6), tick=False) + + ohlcv_mock.reset_mock() + res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp()) + assert ohlcv_mock.call_count == 1 + assert len(ohlcv_mock.call_args_list[0][0][0]) == 1 + + # Expire 5m and 1h cache + time_machine.move_to(start + timedelta(hours=2), tick=False) + + ohlcv_mock.reset_mock() + res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp()) + assert ohlcv_mock.call_count == 1 + assert len(ohlcv_mock.call_args_list[0][0][0]) == 2 + + # Expire all caches + time_machine.move_to(start + timedelta(days=1, hours=2), tick=False) + + ohlcv_mock.reset_mock() + res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp()) + assert ohlcv_mock.call_count == 1 + assert len(ohlcv_mock.call_args_list[0][0][0]) == 5 + assert ohlcv_mock.call_args_list[0][0][0] == pairs + + @pytest.mark.parametrize("exchange_name", EXCHANGES) async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_name): ohlcv = [ From 4bcf2c423a1d478bfa54b85bcdef077086ee23a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Feb 2024 16:27:43 +0100 Subject: [PATCH 23/54] Don't tick on ttl cache --- tests/utils/test_periodiccache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/test_periodiccache.py b/tests/utils/test_periodiccache.py index df05de4ef..a8931d6a2 100644 --- a/tests/utils/test_periodiccache.py +++ b/tests/utils/test_periodiccache.py @@ -5,7 +5,7 @@ from freqtrade.util import PeriodicCache def test_ttl_cache(): - with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + with time_machine.travel("2021-09-01 05:00:00 +00:00", tick=False) as t: cache = PeriodicCache(5, ttl=60) cache1h = PeriodicCache(5, ttl=3600) From 78d8a4df2ea41cab6e2c9932091267f1d9316518 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Feb 2024 16:29:52 +0100 Subject: [PATCH 24/54] Use "ohlcv_with_cache" for further pairlists --- freqtrade/plugins/pairlist/VolatilityFilter.py | 5 +---- freqtrade/plugins/pairlist/rangestabilityfilter.py | 6 +----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 794df5449..b6ce1b9a2 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -104,10 +104,7 @@ class VolatilityFilter(IPairList): since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days)) # Get all candles - candles = {} - if needed_pairs: - candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, - cache=False) + candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms) if self._enabled: for p in deepcopy(pairlist): diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index e04772e9c..f2cf4d486 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -101,11 +101,7 @@ class RangeStabilityFilter(IPairList): (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache] since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days - 1)) - # Get all candles - candles = {} - if needed_pairs: - candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, - cache=False) + candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms) if self._enabled: for p in deepcopy(pairlist): From ebd439cdd10ede780da071f0829b8b23fd9cef18 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Feb 2024 16:41:10 +0100 Subject: [PATCH 25/54] Remove unused import --- freqtrade/exchange/exchange.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6dbb38bd8..ee3ca05cf 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -6,7 +6,6 @@ import asyncio import inspect import logging import signal -from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta, timezone from math import floor From a5d1ae31915f401e8b0031d99eacac4e181f516d Mon Sep 17 00:00:00 2001 From: William Wong <46506352+tar-xz@users.noreply.github.com> Date: Sun, 18 Feb 2024 03:21:50 +0800 Subject: [PATCH 26/54] docs: Update sql_cheatsheet.md --- docs/sql_cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 67c081d4c..a0c5c8da1 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -109,7 +109,7 @@ Freqtrade does not depend or install any additional database driver. Please refe The following systems have been tested and are known to work with freqtrade: * sqlite (default) -* PostgreSQL) +* PostgreSQL * MariaDB !!! Warning From 3250f42257777f076b72bf7bf8966940f9bfc618 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 11:21:34 +0100 Subject: [PATCH 27/54] Improve validate_exchange returns now both required and optional dependencies --- freqtrade/exchange/common.py | 1 + freqtrade/exchange/exchange_utils.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 72ad774b6..d04241e29 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -86,6 +86,7 @@ EXCHANGE_HAS_OPTIONAL = [ # 'fetchPositions', # Futures trading # 'fetchLeverageTiers', # Futures initialization # 'fetchMarketLeverageTiers', # Futures initialization + # 'fetchOpenOrder', 'fetchClosedOrder', # replacement for fetchOrder # 'fetchOpenOrders', 'fetchClosedOrders', # 'fetchOrders', # Refinding balance... ] diff --git a/freqtrade/exchange/exchange_utils.py b/freqtrade/exchange/exchange_utils.py index 98e05bf7a..f8da47fee 100644 --- a/freqtrade/exchange/exchange_utils.py +++ b/freqtrade/exchange/exchange_utils.py @@ -40,21 +40,30 @@ def available_exchanges(ccxt_module: Optional[CcxtModuleType] = None) -> List[st def validate_exchange(exchange: str) -> Tuple[bool, str]: + """ + returns: can_use, reason + with Reason including both missing and missing_opt + """ ex_mod = getattr(ccxt, exchange.lower())() + result = True + reason = '' if not ex_mod or not ex_mod.has: return False, '' missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True] if missing: - return False, f"missing: {', '.join(missing)}" + result = False + reason += f"missing: {', '.join(missing)}" missing_opt = [k for k in EXCHANGE_HAS_OPTIONAL if not ex_mod.has.get(k)] if exchange.lower() in BAD_EXCHANGES: - return False, BAD_EXCHANGES.get(exchange.lower(), '') - if missing_opt: - return True, f"missing opt: {', '.join(missing_opt)}" + result = False + reason = BAD_EXCHANGES.get(exchange.lower(), '') - return True, '' + if missing_opt: + reason += f"{'. ' if reason else ''}missing opt: {', '.join(missing_opt)}. " + + return result, reason def _build_exchange_list_entry( From e06b70eb0530e81a8e6bf503a94c4bef45884a9d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 11:40:50 +0100 Subject: [PATCH 28/54] Add log message for Bybit accout type --- freqtrade/exchange/bybit.py | 16 +++++++++++++--- tests/exchange/test_bybit.py | 21 ++++++++++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py index e7c463140..259858802 100644 --- a/freqtrade/exchange/bybit.py +++ b/freqtrade/exchange/bybit.py @@ -25,6 +25,7 @@ class Bybit(Exchange): officially supported by the Freqtrade development team. So some features may still not work as expected. """ + unified_account = False _ft_has: Dict = { "ohlcv_candle_limit": 1000, @@ -82,9 +83,18 @@ class Bybit(Exchange): Must be overridden in child methods if required. """ try: - if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']: - position_mode = self._api.set_position_mode(False) - self._log_exchange_response('set_position_mode', position_mode) + if not self._config['dry_run']: + if self.trading_mode == TradingMode.FUTURES: + position_mode = self._api.set_position_mode(False) + self._log_exchange_response('set_position_mode', position_mode) + is_unified = self._api.is_unified_enabled() + # Returns a tuple of bools, first for margin, second for Account + if is_unified and len(is_unified) > 1 and is_unified[1]: + self.unified_account = True + logger.info("Bybit: Unified account.") + else: + self.unified_account = False + logger.info("Bybit: Standard account.") except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: diff --git a/tests/exchange/test_bybit.py b/tests/exchange/test_bybit.py index f7383934b..74a490aa9 100644 --- a/tests/exchange/test_bybit.py +++ b/tests/exchange/test_bybit.py @@ -3,18 +3,33 @@ from unittest.mock import MagicMock from freqtrade.enums.marginmode import MarginMode from freqtrade.enums.tradingmode import TradingMode -from tests.conftest import EXMS, get_mock_coro, get_patched_exchange +from tests.conftest import EXMS, get_mock_coro, get_patched_exchange, log_has from tests.exchange.test_exchange import ccxt_exceptionhandlers -def test_additional_exchange_init_bybit(default_conf, mocker): +def test_additional_exchange_init_bybit(default_conf, mocker, caplog): default_conf['dry_run'] = False default_conf['trading_mode'] = TradingMode.FUTURES default_conf['margin_mode'] = MarginMode.ISOLATED api_mock = MagicMock() api_mock.set_position_mode = MagicMock(return_value={"dualSidePosition": False}) - get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock) + api_mock.is_unified_enabled = MagicMock(return_value=[False, False]) + + exchange = get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock) assert api_mock.set_position_mode.call_count == 1 + assert api_mock.is_unified_enabled.call_count == 1 + assert exchange.unified_account is False + + assert log_has("Bybit: Standard account.", caplog) + + api_mock.set_position_mode.reset_mock() + api_mock.is_unified_enabled = MagicMock(return_value=[False, True]) + exchange = get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock) + assert api_mock.set_position_mode.call_count == 1 + assert api_mock.is_unified_enabled.call_count == 1 + assert exchange.unified_account is True + + assert log_has("Bybit: Unified account.", caplog) ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'bybit', "additional_exchange_init", "set_position_mode") From 583b2fc690f2461126d7b14b6af560b87537e928 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 11:44:54 +0100 Subject: [PATCH 29/54] Fail if unified account is detected. --- freqtrade/exchange/bybit.py | 2 ++ tests/exchange/test_bybit.py | 15 ++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py index 259858802..63047066a 100644 --- a/freqtrade/exchange/bybit.py +++ b/freqtrade/exchange/bybit.py @@ -92,6 +92,8 @@ class Bybit(Exchange): if is_unified and len(is_unified) > 1 and is_unified[1]: self.unified_account = True logger.info("Bybit: Unified account.") + raise OperationalException("Bybit: Unified account is not supported. " + "Please use a standard (sub)account.") else: self.unified_account = False logger.info("Bybit: Standard account.") diff --git a/tests/exchange/test_bybit.py b/tests/exchange/test_bybit.py index 74a490aa9..fb7d7a120 100644 --- a/tests/exchange/test_bybit.py +++ b/tests/exchange/test_bybit.py @@ -1,8 +1,11 @@ from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock +import pytest + from freqtrade.enums.marginmode import MarginMode from freqtrade.enums.tradingmode import TradingMode +from freqtrade.exceptions import OperationalException from tests.conftest import EXMS, get_mock_coro, get_patched_exchange, log_has from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -24,12 +27,14 @@ def test_additional_exchange_init_bybit(default_conf, mocker, caplog): api_mock.set_position_mode.reset_mock() api_mock.is_unified_enabled = MagicMock(return_value=[False, True]) - exchange = get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock) - assert api_mock.set_position_mode.call_count == 1 - assert api_mock.is_unified_enabled.call_count == 1 - assert exchange.unified_account is True - + with pytest.raises(OperationalException, match=r"Bybit: Unified account is not supported.*"): + get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock) assert log_has("Bybit: Unified account.", caplog) + # exchange = get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock) + # assert api_mock.set_position_mode.call_count == 1 + # assert api_mock.is_unified_enabled.call_count == 1 + # assert exchange.unified_account is True + ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'bybit', "additional_exchange_init", "set_position_mode") From 61e09ac719f9c1bffb2cb082bb45dd75657154fa Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 16:07:03 +0100 Subject: [PATCH 30/54] Update telegram help with new wording --- freqtrade/rpc/telegram.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index fcc61b5e4..904b1fdbc 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1364,7 +1364,7 @@ class Telegram(RPCHandler): @authorized_only async def _enter_tag_performance(self, update: Update, context: CallbackContext) -> None: """ - Handler for /buys PAIR . + Handler for /entries PAIR . Shows a performance statistic from finished trades :param bot: telegram bot :param update: message update @@ -1396,7 +1396,7 @@ class Telegram(RPCHandler): @authorized_only async def _exit_reason_performance(self, update: Update, context: CallbackContext) -> None: """ - Handler for /sells. + Handler for /exits. Shows a performance statistic from finished trades :param bot: telegram bot :param update: message update @@ -1676,8 +1676,8 @@ class Telegram(RPCHandler): " *table :* `will display trades in a table`\n" " `pending buy orders are marked with an asterisk (*)`\n" " `pending sell orders are marked with a double asterisk (**)`\n" - "*/buys :* `Shows the enter_tag performance`\n" - "*/sells :* `Shows the exit reason performance`\n" + "*/entries :* `Shows the enter_tag performance`\n" + "*/exits :* `Shows the exit reason performance`\n" "*/mix_tags :* `Shows combined entry tag + exit reason performance`\n" "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n" "*/profit []:* `Lists cumulative profit from all finished trades, " From 69a0f4c465fa4f45ef8d4271de85839897c70f05 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 16:09:59 +0100 Subject: [PATCH 31/54] Fix bybit spot live tests --- tests/exchange_online/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/exchange_online/conftest.py b/tests/exchange_online/conftest.py index a613ae586..f8cd8f413 100644 --- a/tests/exchange_online/conftest.py +++ b/tests/exchange_online/conftest.py @@ -324,7 +324,8 @@ def get_futures_exchange(exchange_name, exchange_conf, class_mocker): @pytest.fixture(params=EXCHANGES, scope="class") -def exchange(request, exchange_conf): +def exchange(request, exchange_conf, class_mocker): + class_mocker.patch('freqtrade.exchange.bybit.Bybit.additional_exchange_init') yield from get_exchange(request.param, exchange_conf) From 4c3879cb57b0aaad66e7b1767fe6c0b58de16fc3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 03:11:21 +0000 Subject: [PATCH 32/54] Bump the types group with 1 update Bumps the types group with 1 update: [types-requests](https://github.com/python/typeshed). Updates `types-requests` from 2.31.0.20240125 to 2.31.0.20240218 - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch dependency-group: types ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f0095bffa..bee5140f6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -26,6 +26,6 @@ nbconvert==7.16.0 # mypy types types-cachetools==5.3.0.7 types-filelock==3.2.7 -types-requests==2.31.0.20240125 +types-requests==2.31.0.20240218 types-tabulate==0.9.0.20240106 types-python-dateutil==2.8.19.20240106 From 0979d0b6e464b15c67e79d10530b846d1c1e65d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 03:11:53 +0000 Subject: [PATCH 33/54] Bump the pytest group with 1 update Bumps the pytest group with 1 update: [pytest](https://github.com/pytest-dev/pytest). Updates `pytest` from 8.0.0 to 8.0.1 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.0...8.0.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch dependency-group: pytest ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f0095bffa..a70af1fb9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ coveralls==3.3.1 ruff==0.2.1 mypy==1.8.0 pre-commit==3.6.1 -pytest==8.0.0 +pytest==8.0.1 pytest-asyncio==0.23.5 pytest-cov==4.1.0 pytest-mock==3.12.0 From 8675f86d14af7021eb6386ad0d997d07bbc9dfb6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 03:11:57 +0000 Subject: [PATCH 34/54] Bump urllib3 from 2.2.0 to 2.2.1 Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.2.0 to 2.2.1. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.2.0...2.2.1) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c30036e5a..72e342523 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ httpx>=0.24.1 arrow==1.3.0 cachetools==5.3.2 requests==2.31.0 -urllib3==2.2.0 +urllib3==2.2.1 jsonschema==4.21.1 TA-Lib==0.4.28 technical==1.4.3 From f361824b15b5e7bd7c83774ac420c5f825a1d7a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 03:12:05 +0000 Subject: [PATCH 35/54] Bump cryptography from 42.0.2 to 42.0.3 Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.2 to 42.0.3. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.2...42.0.3) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c30036e5a..52ae42c60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ pandas==2.1.4 pandas-ta==0.3.14b ccxt==4.2.42 -cryptography==42.0.2 +cryptography==42.0.3 aiohttp==3.9.3 SQLAlchemy==2.0.26 python-telegram-bot==20.8 From c966f83147da6f4aebd394cfd90fc097b4213b80 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 03:12:17 +0000 Subject: [PATCH 36/54] Bump plotly from 5.18.0 to 5.19.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.18.0 to 5.19.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.18.0...v5.19.0) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 8900bf1f9..af746ef98 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.18.0 +plotly==5.19.0 From 4241db2fe25b7fc37282ec8e47acc08aed10b4b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 03:12:37 +0000 Subject: [PATCH 37/54] Bump scikit-learn from 1.4.0 to 1.4.1.post1 Bumps [scikit-learn](https://github.com/scikit-learn/scikit-learn) from 1.4.0 to 1.4.1.post1. - [Release notes](https://github.com/scikit-learn/scikit-learn/releases) - [Commits](https://github.com/scikit-learn/scikit-learn/compare/1.4.0...1.4.1.post1) --- updated-dependencies: - dependency-name: scikit-learn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-freqai.txt | 2 +- requirements-hyperopt.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 848b6d920..3719f7d57 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -3,7 +3,7 @@ -r requirements-plot.txt # Required for freqai -scikit-learn==1.4.0 +scikit-learn==1.4.1.post1 joblib==1.3.2 catboost==1.2.2; 'arm' not in platform_machine and python_version < '3.12' lightgbm==4.3.0 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index b961b3b04..5347adf9c 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -3,6 +3,6 @@ # Required for hyperopt scipy==1.12.0 -scikit-learn==1.4.0 +scikit-learn==1.4.1.post1 ft-scikit-optimize==0.9.2 filelock==3.13.1 From edb5431a778e367c029d4f163f23288a8ee8cee6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 03:13:07 +0000 Subject: [PATCH 38/54] Bump orjson from 3.9.13 to 3.9.14 Bumps [orjson](https://github.com/ijl/orjson) from 3.9.13 to 3.9.14. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.9.13...3.9.14) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c30036e5a..73688c870 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ py_find_1st==1.1.6 # Load ticker files 30% faster python-rapidjson==1.14 # Properly format api responses -orjson==3.9.13 +orjson==3.9.14 # Notify systemd sdnotify==0.3.2 From 549b9f62fdc234c6dc43e7c0875a8d5c8438a905 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 03:13:12 +0000 Subject: [PATCH 39/54] Bump tensorboard from 2.15.2 to 2.16.2 Bumps [tensorboard](https://github.com/tensorflow/tensorboard) from 2.15.2 to 2.16.2. - [Release notes](https://github.com/tensorflow/tensorboard/releases) - [Changelog](https://github.com/tensorflow/tensorboard/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorboard/compare/2.15.2...2.16.2) --- updated-dependencies: - dependency-name: tensorboard dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-freqai.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 848b6d920..1df9ad416 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -8,5 +8,5 @@ joblib==1.3.2 catboost==1.2.2; 'arm' not in platform_machine and python_version < '3.12' lightgbm==4.3.0 xgboost==2.0.3 -tensorboard==2.15.2 +tensorboard==2.16.2 datasieve==0.1.7 From 381576b8f148517bcc9fc74023599122b2c3a156 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 05:30:18 +0000 Subject: [PATCH 40/54] Bump pre-commit from 3.6.1 to 3.6.2 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 3.6.1 to 3.6.2. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v3.6.1...v3.6.2) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index a70af1fb9..13d699a16 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,7 +9,7 @@ coveralls==3.3.1 ruff==0.2.1 mypy==1.8.0 -pre-commit==3.6.1 +pre-commit==3.6.2 pytest==8.0.1 pytest-asyncio==0.23.5 pytest-cov==4.1.0 From 6aa1bbf574f417d122475bdf97a8ed057b484e6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 05:31:00 +0000 Subject: [PATCH 41/54] Bump sqlalchemy from 2.0.26 to 2.0.27 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 2.0.26 to 2.0.27. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d18950d73..09933bffa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ pandas-ta==0.3.14b ccxt==4.2.42 cryptography==42.0.3 aiohttp==3.9.3 -SQLAlchemy==2.0.26 +SQLAlchemy==2.0.27 python-telegram-bot==20.8 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.24.1 From 66f48391014a3005b321635e8b8a801b3f51c991 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Feb 2024 07:02:37 +0100 Subject: [PATCH 42/54] Further increase test coverate of max_drawdown --- tests/data/test_btanalysis.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index c1b007e77..554ee261a 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -455,6 +455,13 @@ def test_calculate_max_drawdown2(): with pytest.raises(ValueError, match='No losing trade, therefore no drawdown.'): calculate_max_drawdown(df, date_col='open_date', value_col='profit') + df1 = DataFrame(zip(values[:5], dates[:5]), columns=['profit', 'open_date']) + df1.loc[:, 'profit'] = df1['profit'] * -1 + # No winning trade ... + drawdown, hdate, ldate, hval, lval, drawdown_rel = calculate_max_drawdown( + df1, date_col='open_date', value_col='profit') + assert drawdown == 0.043965 + @pytest.mark.parametrize('profits,relative,highd,lowd,result,result_rel', [ ([0.0, -500.0, 500.0, 10000.0, -1000.0], False, 3, 4, 1000.0, 0.090909), From 39941a7ac04da6b8aa9fabe947ed2ca266ef1b27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Feb 2024 07:09:23 +0100 Subject: [PATCH 43/54] Improve formatting in drawdown calc --- freqtrade/data/metrics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py index b37e0bb19..738129939 100644 --- a/freqtrade/data/metrics.py +++ b/freqtrade/data/metrics.py @@ -143,8 +143,10 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date' starting_balance=starting_balance ) - idxmin = max_drawdown_df['drawdown_relative'].idxmax() if relative \ - else max_drawdown_df['drawdown'].idxmin() + idxmin = ( + max_drawdown_df['drawdown_relative'].idxmax() + if relative else max_drawdown_df['drawdown'].idxmin() + ) if idxmin == 0: raise ValueError("No losing trade, therefore no drawdown.") high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] From a200b5524b88ccdb3e9f5c3a52c93c3cff367521 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Feb 2024 07:18:56 +0100 Subject: [PATCH 44/54] Update sqlalchemy in pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a1aa00f07..f843b6ebe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: - types-requests==2.31.0.20240125 - types-tabulate==0.9.0.20240106 - types-python-dateutil==2.8.19.20240106 - - SQLAlchemy==2.0.26 + - SQLAlchemy==2.0.27 # stages: [push] - repo: https://github.com/pycqa/isort From 82876570a339882987cd441a7a0b459ae0569efd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 07:14:01 +0000 Subject: [PATCH 45/54] Bump ruff from 0.2.1 to 0.2.2 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.2.1 to 0.2.2. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/v0.2.1...v0.2.2) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 13d699a16..c606f219d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.2.1 +ruff==0.2.2 mypy==1.8.0 pre-commit==3.6.2 pytest==8.0.1 From 434b8a423cda08ada2d580ee58ab70204f25f772 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Feb 2024 09:33:09 +0100 Subject: [PATCH 46/54] bump types-requests pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a1aa00f07..b73c2e5cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: additional_dependencies: - types-cachetools==5.3.0.7 - types-filelock==3.2.7 - - types-requests==2.31.0.20240125 + - types-requests==2.31.0.20240218 - types-tabulate==0.9.0.20240106 - types-python-dateutil==2.8.19.20240106 - SQLAlchemy==2.0.26 From 00bde70f73e9363fdfe3e0f6b305b69cb959a6c0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Feb 2024 19:14:44 +0100 Subject: [PATCH 47/54] Fix / improve styling in test class --- tests/rpc/test_rpc_apiserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index e441b127b..1e008d98e 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -180,7 +180,9 @@ def test_api_auth(): def test_api_ws_auth(botclient): ftbot, client = botclient - def url(token): return f"/api/v1/message/ws?token={token}" + + def url(token): + return f"/api/v1/message/ws?token={token}" bad_token = "bad-ws_token" with pytest.raises(WebSocketDisconnect): From f6e2030bf2c22ca4bf775df8efee65ea81aeff93 Mon Sep 17 00:00:00 2001 From: xmatthias <5024695+xmatthias@users.noreply.github.com> Date: Tue, 20 Feb 2024 03:03:46 +0000 Subject: [PATCH 48/54] chore: update pre-commit hooks --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f9865be7b..842c87976 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: - repo: https://github.com/charliermarsh/ruff-pre-commit # Ruff version. - rev: 'v0.2.1' + rev: 'v0.2.2' hooks: - id: ruff From 65af7750e6cbb5ce9c026036d2dee1bf0ef2a20e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 13:00:43 +0100 Subject: [PATCH 49/54] Add fetch_order_emulated to support exchanges without proper fetch_order method --- freqtrade/exchange/exchange.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 1df51ed90..2872e603e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1260,11 +1260,43 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def fetch_order_emulated(self, order_id: str, pair: str, params: Dict) -> Dict: + """ + Emulated fetch_order if the exchange doesn't support fetch_order, but requires separate + calls for open and closed orders. + """ + try: + order = self._api.fetch_open_order(order_id, pair, params=params) + self._log_exchange_response('fetch_open_order', order) + order = self._order_contracts_to_amount(order) + return order + except ccxt.OrderNotFound: + try: + order = self._api.fetch_closed_order(order_id, pair, params=params) + self._log_exchange_response('fetch_closed_order', order) + order = self._order_contracts_to_amount(order) + return order + except ccxt.OrderNotFound as e: + raise RetryableOrderError( + f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict: if self._config['dry_run']: return self.fetch_dry_run_order(order_id) try: + if not self.exchange_has('fetchOrder'): + return self.fetch_order_emulated(order_id, pair, params) order = self._api.fetch_order(order_id, pair, params=params) self._log_exchange_response('fetch_order', order) order = self._order_contracts_to_amount(order) From 3497f7946e89608551f3873e2eb5ecad93aa850e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 13:09:40 +0100 Subject: [PATCH 50/54] Add test for fetch_order_emulated --- tests/exchange/test_exchange.py | 66 +++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ef41a6eb0..077f1f8f0 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3391,6 +3391,72 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog): order_id='_', pair='TKN/BTC') +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_fetch_order_emulated(default_conf, mocker, exchange_name, caplog): + default_conf['dry_run'] = True + default_conf['exchange']['log_responses'] = True + order = MagicMock() + order.myid = 123 + order.symbol = 'TKN/BTC' + + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch(f'{EXMS}.exchange_has', return_value=False) + exchange._dry_run_open_orders['X'] = order + # Dry run - regular fetch_order behavior + assert exchange.fetch_order('X', 'TKN/BTC').myid == 123 + + with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'): + exchange.fetch_order('Y', 'TKN/BTC') + + default_conf['dry_run'] = False + mocker.patch(f'{EXMS}.exchange_has', return_value=False) + api_mock = MagicMock() + api_mock.fetch_open_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + api_mock.fetch_closed_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + assert exchange.fetch_order( + 'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'} + assert log_has( + ("API fetch_open_order: {\'id\': \'123\', \'amount\': 2, \'symbol\': \'TKN/BTC\'}" + ), + caplog + ) + assert api_mock.fetch_open_order.call_count == 1 + assert api_mock.fetch_closed_order.call_count == 0 + caplog.clear() + + # open_order doesn't find order + api_mock.fetch_open_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found")) + api_mock.fetch_closed_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + assert exchange.fetch_order( + 'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'} + assert log_has( + ("API fetch_closed_order: {\'id\': \'123\', \'amount\': 2, \'symbol\': \'TKN/BTC\'}" + ), + caplog + ) + assert api_mock.fetch_open_order.call_count == 1 + assert api_mock.fetch_closed_order.call_count == 1 + caplog.clear() + + with pytest.raises(InvalidOrderException): + api_mock.fetch_open_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + api_mock.fetch_closed_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + exchange.fetch_order(order_id='_', pair='TKN/BTC') + assert api_mock.fetch_open_order.call_count == 1 + + api_mock.fetch_open_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + + ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, + 'fetch_order_emulated', 'fetch_open_order', + retries=1, + order_id='_', pair='TKN/BTC', params={}) + + @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_fetch_stoploss_order(default_conf, mocker, exchange_name): From f53c019d2afa80f33b96e2fad61290d9c01a46d8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 15:14:07 +0100 Subject: [PATCH 51/54] Update "exchange_has" validation with new fallbacks --- freqtrade/exchange/common.py | 15 ++++++++------- freqtrade/exchange/exchange_utils.py | 6 +++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index d04241e29..06ae21001 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -60,16 +60,17 @@ SUPPORTED_EXCHANGES = [ 'okx', ] -EXCHANGE_HAS_REQUIRED = [ +# either the main, or replacement methods (array) is required +EXCHANGE_HAS_REQUIRED = { # Required / private - 'fetchOrder', - 'cancelOrder', - 'createOrder', - 'fetchBalance', + 'fetchOrder': ['fetchOpenOrder', 'fetchClosedOrder'], + 'cancelOrder': [], + 'createOrder': [], + 'fetchBalance': [], # Public endpoints - 'fetchOHLCV', -] + 'fetchOHLCV': [], +} EXCHANGE_HAS_OPTIONAL = [ # Private diff --git a/freqtrade/exchange/exchange_utils.py b/freqtrade/exchange/exchange_utils.py index f8da47fee..f4dc3a721 100644 --- a/freqtrade/exchange/exchange_utils.py +++ b/freqtrade/exchange/exchange_utils.py @@ -49,7 +49,11 @@ def validate_exchange(exchange: str) -> Tuple[bool, str]: reason = '' if not ex_mod or not ex_mod.has: return False, '' - missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True] + missing = [ + k for k, v in EXCHANGE_HAS_REQUIRED.items() + if ex_mod.has.get(k) is not True + and not (all(ex_mod.has.get(x) for x in v)) + ] if missing: result = False reason += f"missing: {', '.join(missing)}" From 411f60647649ddb0833180dc5df7554939462005 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 15:39:13 +0100 Subject: [PATCH 52/54] Fix some tests due to new method --- tests/exchange/test_bybit.py | 1 + tests/exchange/test_exchange.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/exchange/test_bybit.py b/tests/exchange/test_bybit.py index fb7d7a120..556547d88 100644 --- a/tests/exchange/test_bybit.py +++ b/tests/exchange/test_bybit.py @@ -131,6 +131,7 @@ def test_bybit_fetch_order_canceled_empty(default_conf_usdt, mocker): 'amount': 20.0, }) + mocker.patch(f"{EXMS}.exchange_has", return_value=True) exchange = get_patched_exchange(mocker, default_conf_usdt, api_mock, id='bybit') res = exchange.fetch_order('123', 'BTC/USDT') diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 077f1f8f0..5c4879a32 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3237,6 +3237,7 @@ def test_is_cancel_order_result_suitable(mocker, default_conf, exchange_name, or def test_cancel_order_with_result(default_conf, mocker, exchange_name, corder, call_corder, call_forder): default_conf['dry_run'] = False + mocker.patch(f"{EXMS}.exchange_has", return_value=True) api_mock = MagicMock() api_mock.cancel_order = MagicMock(return_value=corder) api_mock.fetch_order = MagicMock(return_value={}) @@ -3250,6 +3251,7 @@ def test_cancel_order_with_result(default_conf, mocker, exchange_name, corder, @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_cancel_order_with_result_error(default_conf, mocker, exchange_name, caplog): default_conf['dry_run'] = False + mocker.patch(f"{EXMS}.exchange_has", return_value=True) api_mock = MagicMock() api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order")) api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order")) @@ -3347,6 +3349,7 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog): order.myid = 123 order.symbol = 'TKN/BTC' + mocker.patch(f"{EXMS}.exchange_has", return_value=True) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) exchange._dry_run_open_orders['X'] = order assert exchange.fetch_order('X', 'TKN/BTC').myid == 123 @@ -3412,8 +3415,10 @@ def test_fetch_order_emulated(default_conf, mocker, exchange_name, caplog): default_conf['dry_run'] = False mocker.patch(f'{EXMS}.exchange_has', return_value=False) api_mock = MagicMock() - api_mock.fetch_open_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) - api_mock.fetch_closed_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + api_mock.fetch_open_order = MagicMock( + return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + api_mock.fetch_closed_order = MagicMock( + return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) assert exchange.fetch_order( 'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'} @@ -3428,7 +3433,8 @@ def test_fetch_order_emulated(default_conf, mocker, exchange_name, caplog): # open_order doesn't find order api_mock.fetch_open_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found")) - api_mock.fetch_closed_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + api_mock.fetch_closed_order = MagicMock( + return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) assert exchange.fetch_order( 'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'} @@ -3461,6 +3467,7 @@ def test_fetch_order_emulated(default_conf, mocker, exchange_name, caplog): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_fetch_stoploss_order(default_conf, mocker, exchange_name): default_conf['dry_run'] = True + mocker.patch(f"{EXMS}.exchange_has", return_value=True) order = MagicMock() order.myid = 123 exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) From b3ba2cee1744141fd0ef252ae731d27a9e65a8df Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Feb 2024 06:17:13 +0100 Subject: [PATCH 53/54] Bump ccxt to 4.2.47 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 17ebe8867..1efe0b7da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.26.4 pandas==2.1.4 pandas-ta==0.3.14b -ccxt==4.2.42 +ccxt==4.2.47 cryptography==42.0.3 aiohttp==3.9.3 SQLAlchemy==2.0.27 diff --git a/setup.py b/setup.py index 3b92b9dd7..38f0f9a78 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ setup( ], install_requires=[ # from requirements.txt - 'ccxt>=4.2.15', + 'ccxt>=4.2.47', 'SQLAlchemy>=2.0.6', 'python-telegram-bot>=20.1', 'arrow>=1.0.0', From 0199e7d3d8dfe0046be307eabe70b58a6ef1e680 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Feb 2024 06:30:10 +0100 Subject: [PATCH 54/54] Add type-hint to exchange_has dict --- freqtrade/exchange/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 06ae21001..8909ef5ff 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -2,7 +2,7 @@ import asyncio import logging import time from functools import wraps -from typing import Any, Callable, Optional, TypeVar, cast, overload +from typing import Any, Callable, Dict, List, Optional, TypeVar, cast, overload from freqtrade.constants import ExchangeConfig from freqtrade.exceptions import DDosProtection, RetryableOrderError, TemporaryError @@ -61,7 +61,7 @@ SUPPORTED_EXCHANGES = [ ] # either the main, or replacement methods (array) is required -EXCHANGE_HAS_REQUIRED = { +EXCHANGE_HAS_REQUIRED: Dict[str, List[str]] = { # Required / private 'fetchOrder': ['fetchOpenOrder', 'fetchClosedOrder'], 'cancelOrder': [],