From dc92787f1dc841440c80990af6bc14da5f47a2b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2024 11:52:41 +0200 Subject: [PATCH 1/6] Fix gone-wrong hyperopt fix closes #10192 --- .../optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py index e5ec6075e..61e2a6d32 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py @@ -32,4 +32,6 @@ class ProfitDrawDownHyperOptLoss(IHyperOptLoss): except ValueError: relative_account_drawdown = 0 - return -1 * (total_profit - (relative_account_drawdown * total_profit) / DRAWDOWN_MULT) + return -1 * ( + total_profit - (relative_account_drawdown * total_profit) * (1 - DRAWDOWN_MULT) + ) From ec0f6cb2462f9793f984e58511c517151f85973e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2024 20:05:29 +0200 Subject: [PATCH 2/6] Add Properties for canceled orders to trade_model --- freqtrade/persistence/trade_model.py | 26 +++++++++++++++++++++++++- tests/persistence/test_persistence.py | 16 ++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 28080cc20..7d803ea0f 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -957,7 +957,24 @@ class LocalTrade: def update_order(self, order: Dict) -> None: Order.update_orders(self.orders, order) - def get_canceled_exit_order_count(self) -> int: + @property + def fully_canceled_entry_order_count(self) -> int: + """ + Get amount of failed exiting orders + assumes full exits. + """ + return len( + [ + o + for o in self.orders + if o.ft_order_side == self.entry_side + and o.status in CANCELED_EXCHANGE_STATES + and o.filled == 0 + ] + ) + + @property + def canceled_exit_order_count(self) -> int: """ Get amount of failed exiting orders assumes full exits. @@ -970,6 +987,13 @@ class LocalTrade: ] ) + def get_canceled_exit_order_count(self) -> int: + """ + Get amount of failed exiting orders + assumes full exits. + """ + return self.canceled_exit_order_count + def _calc_open_trade_value(self, amount: float, open_rate: float) -> float: """ Calculate the open_rate including open_fee. diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index a30761486..0545ac861 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -1961,9 +1961,25 @@ def test_get_canceled_exit_order_count(fee, is_short): trade = Trade.get_trades([Trade.pair == "ETC/BTC"]).first() # No canceled order. assert trade.get_canceled_exit_order_count() == 0 + # Property returns the same result + assert trade.canceled_exit_order_count == 0 trade.orders[-1].status = "canceled" assert trade.get_canceled_exit_order_count() == 1 + assert trade.canceled_exit_order_count == 1 + + +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize("is_short", [True, False]) +def test_fully_canceled_entry_order_count(fee, is_short): + create_mock_trades(fee, is_short=is_short) + trade = Trade.get_trades([Trade.pair == "ETC/BTC"]).first() + # No canceled order. + assert trade.fully_canceled_entry_order_count == 0 + + trade.orders[0].status = "canceled" + trade.orders[0].filled = 0 + assert trade.fully_canceled_entry_order_count == 1 @pytest.mark.usefixtures("init_persistence") From edd92194b0f3fd14687145f228ba4e84f5fef67b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2024 20:06:35 +0200 Subject: [PATCH 3/6] have handle_onexchange_order delete trades if no order filled. --- freqtrade/freqtradebot.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 401de5147..021a946d5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -492,10 +492,11 @@ class FreqtradeBot(LoggingMixin): except ExchangeError: logger.warning(f"Error updating {order.order_id}.") - def handle_onexchange_order(self, trade: Trade): + def handle_onexchange_order(self, trade: Trade) -> bool: """ Try refinding a order that is not in the database. Only used balance disappeared, which would make exiting impossible. + :return: True if the trade was deleted, False otherwise """ try: orders = self.exchange.fetch_orders( @@ -541,6 +542,19 @@ class FreqtradeBot(LoggingMixin): trade.exit_reason = prev_exit_reason total = self.wallets.get_total(trade.base_currency) if trade.base_currency else 0 if total < trade.amount: + if trade.fully_canceled_entry_order_count == len(trade.orders): + logger.warning( + f"Trade only had fully canceled entry orders. " + f"Removing {trade} from database." + ) + + self._notify_enter_cancel( + trade, + order_type=self.strategy.order_types["entry"], + reason=constants.CANCEL_REASON["FULLY_CANCELLED"], + ) + trade.delete() + return True if total > trade.amount * 0.98: logger.warning( f"{trade} has a total of {trade.amount} {trade.base_currency}, " @@ -566,6 +580,7 @@ class FreqtradeBot(LoggingMixin): except Exception: # catching https://github.com/freqtrade/freqtrade/issues/9025 logger.warning("Error finding onexchange order", exc_info=True) + return False # # enter positions / open trades logic and methods @@ -1229,7 +1244,9 @@ class FreqtradeBot(LoggingMixin): f"Not enough {trade.safe_base_currency} in wallet to exit {trade}. " "Trying to recover." ) - self.handle_onexchange_order(trade) + if self.handle_onexchange_order(trade): + # Trade was deleted. Don't continue. + continue try: try: From 9d3073d93079e82569ccd3c52c8b7d67e8f1e30e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 May 2024 08:35:41 +0200 Subject: [PATCH 4/6] Add test for new "fully cancel" logic --- tests/freqtradebot/test_freqtradebot.py | 41 +++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/freqtradebot/test_freqtradebot.py b/tests/freqtradebot/test_freqtradebot.py index 826e7ebf4..4509af207 100644 --- a/tests/freqtradebot/test_freqtradebot.py +++ b/tests/freqtradebot/test_freqtradebot.py @@ -4978,6 +4978,47 @@ def test_handle_onexchange_order_exit(mocker, default_conf_usdt, limit_order, is assert trade.amount == 5.0 +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize("is_short", [False, True]) +def test_handle_onexchange_order_fully_canceled_enter( + mocker, default_conf_usdt, limit_order, is_short, caplog +): + default_conf_usdt["dry_run"] = False + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + + entry_order = limit_order[entry_side(is_short)] + entry_order["status"] = "canceled" + entry_order["filled"] = 0.0 + mock_fo = mocker.patch( + f"{EXMS}.fetch_orders", + return_value=[ + entry_order, + ], + ) + mocker.patch(f"{EXMS}.get_rate", return_value=entry_order["price"]) + + trade = Trade( + pair="ETH/USDT", + fee_open=0.001, + fee_close=0.001, + open_rate=entry_order["price"], + open_date=dt_now(), + stake_amount=entry_order["cost"], + amount=entry_order["amount"], + exchange="binance", + is_short=is_short, + leverage=1, + ) + + trade.orders.append(Order.parse_from_ccxt_object(entry_order, "ADA/USDT", entry_side(is_short))) + Trade.session.add(trade) + assert freqtrade.handle_onexchange_order(trade) is True + assert log_has_re(r"Trade only had fully canceled entry orders\. .*", caplog) + assert mock_fo.call_count == 1 + trades = Trade.get_trades().all() + assert len(trades) == 0 + + def test_get_valid_price(mocker, default_conf_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 32ff3ebb991c031064e7fc9517af19870824549e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 May 2024 09:42:28 +0200 Subject: [PATCH 5/6] Improve handling for immediately canceled orders --- freqtrade/freqtradebot.py | 8 ++++++- tests/freqtradebot/test_freqtradebot.py | 30 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 021a946d5..8d53097a5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1022,7 +1022,13 @@ class FreqtradeBot(LoggingMixin): # Update fees if order is non-opened if order_status in constants.NON_OPEN_EXCHANGE_STATES: - self.update_trade_state(trade, order_id, order) + fully_canceled = self.update_trade_state(trade, order_id, order) + if fully_canceled and mode != "replace": + # Fully canceled orders, may happen with some time in force setups (IOC). + # Should be handled immediately. + self.handle_cancel_enter( + trade, order, order_obj, constants.CANCEL_REASON["TIMEOUT"] + ) return True diff --git a/tests/freqtradebot/test_freqtradebot.py b/tests/freqtradebot/test_freqtradebot.py index 4509af207..e793e3c50 100644 --- a/tests/freqtradebot/test_freqtradebot.py +++ b/tests/freqtradebot/test_freqtradebot.py @@ -1146,6 +1146,36 @@ def test_execute_entry_confirm_error(mocker, default_conf_usdt, fee, limit_order assert not freqtrade.execute_entry(pair, stake_amount) +@pytest.mark.parametrize("is_short", [False, True]) +def test_execute_entry_fully_canceled_on_create( + mocker, default_conf_usdt, fee, limit_order_open, is_short +) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + + mock_hce = mocker.spy(freqtrade, "handle_cancel_enter") + order = limit_order_open[entry_side(is_short)] + pair = "ETH/USDT" + order["symbol"] = pair + order["status"] = "canceled" + order["filled"] = 0.0 + + mocker.patch.multiple( + EXMS, + fetch_ticker=MagicMock(return_value={"bid": 1.9, "ask": 2.2, "last": 1.9}), + create_order=MagicMock(return_value=order), + get_rate=MagicMock(return_value=0.11), + get_min_pair_stake_amount=MagicMock(return_value=1), + get_fee=fee, + ) + stake_amount = 2 + + assert freqtrade.execute_entry(pair, stake_amount) + assert mock_hce.call_count == 1 + # an order that immediately cancels completely should delete the order. + trades = Trade.get_trades().all() + assert len(trades) == 0 + + @pytest.mark.parametrize("is_short", [False, True]) def test_execute_entry_min_leverage(mocker, default_conf_usdt, fee, limit_order, is_short) -> None: default_conf_usdt["trading_mode"] = "futures" From 46e97e5806030d50a815bb803d661f1a7d830d25 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 May 2024 15:49:48 +0200 Subject: [PATCH 6/6] fix htx: Reduce amount of data downloaded on higher timeframes closes #10247 --- freqtrade/exchange/htx.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/exchange/htx.py b/freqtrade/exchange/htx.py index 58eb919bc..f939534e9 100644 --- a/freqtrade/exchange/htx.py +++ b/freqtrade/exchange/htx.py @@ -24,6 +24,10 @@ class Htx(Exchange): "ohlcv_candle_limit": 1000, "l2_limit_range": [5, 10, 20], "l2_limit_range_required": False, + "ohlcv_candle_limit_per_timeframe": { + "1w": 500, + "1M": 500, + }, } def _get_stop_params(self, side: BuySell, ordertype: str, stop_price: float) -> Dict: