From 65af7750e6cbb5ce9c026036d2dee1bf0ef2a20e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 13:00:43 +0100 Subject: [PATCH 1/6] Add fetch_order_emulated to support exchanges without proper fetch_order method --- freqtrade/exchange/exchange.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 1df51ed90..2872e603e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1260,11 +1260,43 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def fetch_order_emulated(self, order_id: str, pair: str, params: Dict) -> Dict: + """ + Emulated fetch_order if the exchange doesn't support fetch_order, but requires separate + calls for open and closed orders. + """ + try: + order = self._api.fetch_open_order(order_id, pair, params=params) + self._log_exchange_response('fetch_open_order', order) + order = self._order_contracts_to_amount(order) + return order + except ccxt.OrderNotFound: + try: + order = self._api.fetch_closed_order(order_id, pair, params=params) + self._log_exchange_response('fetch_closed_order', order) + order = self._order_contracts_to_amount(order) + return order + except ccxt.OrderNotFound as e: + raise RetryableOrderError( + f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict: if self._config['dry_run']: return self.fetch_dry_run_order(order_id) try: + if not self.exchange_has('fetchOrder'): + return self.fetch_order_emulated(order_id, pair, params) order = self._api.fetch_order(order_id, pair, params=params) self._log_exchange_response('fetch_order', order) order = self._order_contracts_to_amount(order) From 3497f7946e89608551f3873e2eb5ecad93aa850e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 13:09:40 +0100 Subject: [PATCH 2/6] Add test for fetch_order_emulated --- tests/exchange/test_exchange.py | 66 +++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ef41a6eb0..077f1f8f0 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3391,6 +3391,72 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog): order_id='_', pair='TKN/BTC') +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_fetch_order_emulated(default_conf, mocker, exchange_name, caplog): + default_conf['dry_run'] = True + default_conf['exchange']['log_responses'] = True + order = MagicMock() + order.myid = 123 + order.symbol = 'TKN/BTC' + + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch(f'{EXMS}.exchange_has', return_value=False) + exchange._dry_run_open_orders['X'] = order + # Dry run - regular fetch_order behavior + assert exchange.fetch_order('X', 'TKN/BTC').myid == 123 + + with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'): + exchange.fetch_order('Y', 'TKN/BTC') + + default_conf['dry_run'] = False + mocker.patch(f'{EXMS}.exchange_has', return_value=False) + api_mock = MagicMock() + api_mock.fetch_open_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + api_mock.fetch_closed_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + assert exchange.fetch_order( + 'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'} + assert log_has( + ("API fetch_open_order: {\'id\': \'123\', \'amount\': 2, \'symbol\': \'TKN/BTC\'}" + ), + caplog + ) + assert api_mock.fetch_open_order.call_count == 1 + assert api_mock.fetch_closed_order.call_count == 0 + caplog.clear() + + # open_order doesn't find order + api_mock.fetch_open_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found")) + api_mock.fetch_closed_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + assert exchange.fetch_order( + 'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'} + assert log_has( + ("API fetch_closed_order: {\'id\': \'123\', \'amount\': 2, \'symbol\': \'TKN/BTC\'}" + ), + caplog + ) + assert api_mock.fetch_open_order.call_count == 1 + assert api_mock.fetch_closed_order.call_count == 1 + caplog.clear() + + with pytest.raises(InvalidOrderException): + api_mock.fetch_open_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + api_mock.fetch_closed_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + exchange.fetch_order(order_id='_', pair='TKN/BTC') + assert api_mock.fetch_open_order.call_count == 1 + + api_mock.fetch_open_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + + ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, + 'fetch_order_emulated', 'fetch_open_order', + retries=1, + order_id='_', pair='TKN/BTC', params={}) + + @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_fetch_stoploss_order(default_conf, mocker, exchange_name): From f53c019d2afa80f33b96e2fad61290d9c01a46d8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 15:14:07 +0100 Subject: [PATCH 3/6] Update "exchange_has" validation with new fallbacks --- freqtrade/exchange/common.py | 15 ++++++++------- freqtrade/exchange/exchange_utils.py | 6 +++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index d04241e29..06ae21001 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -60,16 +60,17 @@ SUPPORTED_EXCHANGES = [ 'okx', ] -EXCHANGE_HAS_REQUIRED = [ +# either the main, or replacement methods (array) is required +EXCHANGE_HAS_REQUIRED = { # Required / private - 'fetchOrder', - 'cancelOrder', - 'createOrder', - 'fetchBalance', + 'fetchOrder': ['fetchOpenOrder', 'fetchClosedOrder'], + 'cancelOrder': [], + 'createOrder': [], + 'fetchBalance': [], # Public endpoints - 'fetchOHLCV', -] + 'fetchOHLCV': [], +} EXCHANGE_HAS_OPTIONAL = [ # Private diff --git a/freqtrade/exchange/exchange_utils.py b/freqtrade/exchange/exchange_utils.py index f8da47fee..f4dc3a721 100644 --- a/freqtrade/exchange/exchange_utils.py +++ b/freqtrade/exchange/exchange_utils.py @@ -49,7 +49,11 @@ def validate_exchange(exchange: str) -> Tuple[bool, str]: reason = '' if not ex_mod or not ex_mod.has: return False, '' - missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True] + missing = [ + k for k, v in EXCHANGE_HAS_REQUIRED.items() + if ex_mod.has.get(k) is not True + and not (all(ex_mod.has.get(x) for x in v)) + ] if missing: result = False reason += f"missing: {', '.join(missing)}" From 411f60647649ddb0833180dc5df7554939462005 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Feb 2024 15:39:13 +0100 Subject: [PATCH 4/6] Fix some tests due to new method --- tests/exchange/test_bybit.py | 1 + tests/exchange/test_exchange.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/exchange/test_bybit.py b/tests/exchange/test_bybit.py index fb7d7a120..556547d88 100644 --- a/tests/exchange/test_bybit.py +++ b/tests/exchange/test_bybit.py @@ -131,6 +131,7 @@ def test_bybit_fetch_order_canceled_empty(default_conf_usdt, mocker): 'amount': 20.0, }) + mocker.patch(f"{EXMS}.exchange_has", return_value=True) exchange = get_patched_exchange(mocker, default_conf_usdt, api_mock, id='bybit') res = exchange.fetch_order('123', 'BTC/USDT') diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 077f1f8f0..5c4879a32 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3237,6 +3237,7 @@ def test_is_cancel_order_result_suitable(mocker, default_conf, exchange_name, or def test_cancel_order_with_result(default_conf, mocker, exchange_name, corder, call_corder, call_forder): default_conf['dry_run'] = False + mocker.patch(f"{EXMS}.exchange_has", return_value=True) api_mock = MagicMock() api_mock.cancel_order = MagicMock(return_value=corder) api_mock.fetch_order = MagicMock(return_value={}) @@ -3250,6 +3251,7 @@ def test_cancel_order_with_result(default_conf, mocker, exchange_name, corder, @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_cancel_order_with_result_error(default_conf, mocker, exchange_name, caplog): default_conf['dry_run'] = False + mocker.patch(f"{EXMS}.exchange_has", return_value=True) api_mock = MagicMock() api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order")) api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order")) @@ -3347,6 +3349,7 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog): order.myid = 123 order.symbol = 'TKN/BTC' + mocker.patch(f"{EXMS}.exchange_has", return_value=True) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) exchange._dry_run_open_orders['X'] = order assert exchange.fetch_order('X', 'TKN/BTC').myid == 123 @@ -3412,8 +3415,10 @@ def test_fetch_order_emulated(default_conf, mocker, exchange_name, caplog): default_conf['dry_run'] = False mocker.patch(f'{EXMS}.exchange_has', return_value=False) api_mock = MagicMock() - api_mock.fetch_open_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) - api_mock.fetch_closed_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + api_mock.fetch_open_order = MagicMock( + return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + api_mock.fetch_closed_order = MagicMock( + return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) assert exchange.fetch_order( 'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'} @@ -3428,7 +3433,8 @@ def test_fetch_order_emulated(default_conf, mocker, exchange_name, caplog): # open_order doesn't find order api_mock.fetch_open_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found")) - api_mock.fetch_closed_order = MagicMock(return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) + api_mock.fetch_closed_order = MagicMock( + return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) assert exchange.fetch_order( 'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'} @@ -3461,6 +3467,7 @@ def test_fetch_order_emulated(default_conf, mocker, exchange_name, caplog): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_fetch_stoploss_order(default_conf, mocker, exchange_name): default_conf['dry_run'] = True + mocker.patch(f"{EXMS}.exchange_has", return_value=True) order = MagicMock() order.myid = 123 exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) From b3ba2cee1744141fd0ef252ae731d27a9e65a8df Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Feb 2024 06:17:13 +0100 Subject: [PATCH 5/6] Bump ccxt to 4.2.47 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 17ebe8867..1efe0b7da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.26.4 pandas==2.1.4 pandas-ta==0.3.14b -ccxt==4.2.42 +ccxt==4.2.47 cryptography==42.0.3 aiohttp==3.9.3 SQLAlchemy==2.0.27 diff --git a/setup.py b/setup.py index 3b92b9dd7..38f0f9a78 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ setup( ], install_requires=[ # from requirements.txt - 'ccxt>=4.2.15', + 'ccxt>=4.2.47', 'SQLAlchemy>=2.0.6', 'python-telegram-bot>=20.1', 'arrow>=1.0.0', From 0199e7d3d8dfe0046be307eabe70b58a6ef1e680 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Feb 2024 06:30:10 +0100 Subject: [PATCH 6/6] Add type-hint to exchange_has dict --- freqtrade/exchange/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 06ae21001..8909ef5ff 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -2,7 +2,7 @@ import asyncio import logging import time from functools import wraps -from typing import Any, Callable, Optional, TypeVar, cast, overload +from typing import Any, Callable, Dict, List, Optional, TypeVar, cast, overload from freqtrade.constants import ExchangeConfig from freqtrade.exceptions import DDosProtection, RetryableOrderError, TemporaryError @@ -61,7 +61,7 @@ SUPPORTED_EXCHANGES = [ ] # either the main, or replacement methods (array) is required -EXCHANGE_HAS_REQUIRED = { +EXCHANGE_HAS_REQUIRED: Dict[str, List[str]] = { # Required / private 'fetchOrder': ['fetchOpenOrder', 'fetchClosedOrder'], 'cancelOrder': [],