From f01e10144784472b91dafdc4d0e711ea77eda62a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2024 10:19:33 +0200 Subject: [PATCH 01/11] feat: extract backtest iteration into generator --- freqtrade/optimize/backtesting.py | 149 ++++++++++++++++-------------- 1 file changed, 81 insertions(+), 68 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c9bdf4c65..b26013a11 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1388,6 +1388,25 @@ class Backtesting: self._process_exit_order(order, trade, current_time, row, pair) return open_trade_count_start + def time_pair_generator( + self, start_date: datetime, end_date: datetime, increment: timedelta, pairs: List[str] + ): + """ + Backtest time and pair generator + """ + current_time = start_date + increment + self.progress.init_step( + BacktestState.BACKTEST, int((end_date - start_date) / self.timeframe_td) + ) + while current_time <= end_date: + is_first = True + for pair in pairs: + yield current_time, pair, is_first + is_first = False + + self.progress.increment() + current_time += increment + def backtest(self, processed: Dict, start_date: datetime, end_date: datetime) -> Dict[str, Any]: """ Implement backtesting functionality @@ -1411,82 +1430,76 @@ class Backtesting: # Indexes per pair, so some pairs are allowed to have a missing start. indexes: Dict = defaultdict(int) - current_time = start_date + self.timeframe_td - self.progress.init_step( - BacktestState.BACKTEST, int((end_date - start_date) / self.timeframe_td) - ) # Loop timerange and get candle for each pair at that point in time - while current_time <= end_date: - open_trade_count_start = LocalTrade.bt_open_open_trade_count - self.check_abort() - strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)( - current_time=current_time - ) - for i, pair in enumerate(data): - row_index = indexes[pair] - row = self.validate_row(data, pair, row_index, current_time) - if not row: - continue + for current_time, pair, is_first in self.time_pair_generator( + start_date, end_date, self.timeframe_td, list(data.keys()) + ): + if is_first: + open_trade_count_start = LocalTrade.bt_open_open_trade_count + self.check_abort() + strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)( + current_time=current_time + ) + row_index = indexes[pair] + row = self.validate_row(data, pair, row_index, current_time) + if not row: + continue - row_index += 1 - indexes[pair] = row_index - self.dataprovider._set_dataframe_max_index(self.required_startup + row_index) - self.dataprovider._set_dataframe_max_date(current_time) - current_detail_time: datetime = row[DATE_IDX].to_pydatetime() - trade_dir: Optional[LongShort] = self.check_for_trade_entry(row) + row_index += 1 + indexes[pair] = row_index + self.dataprovider._set_dataframe_max_index(self.required_startup + row_index) + self.dataprovider._set_dataframe_max_date(current_time) + current_detail_time: datetime = row[DATE_IDX].to_pydatetime() + trade_dir: Optional[LongShort] = self.check_for_trade_entry(row) - if ( - (trade_dir is not None or len(LocalTrade.bt_trades_open_pp[pair]) > 0) - and self.timeframe_detail - and pair in self.detail_data - ): - # Spread out into detail timeframe. - # Should only happen when we are either in a trade for this pair - # or when we got the signal for a new trade. - exit_candle_end = current_detail_time + self.timeframe_td + if ( + (trade_dir is not None or len(LocalTrade.bt_trades_open_pp[pair]) > 0) + and self.timeframe_detail + and pair in self.detail_data + ): + # Spread out into detail timeframe. + # Should only happen when we are either in a trade for this pair + # or when we got the signal for a new trade. + exit_candle_end = current_detail_time + self.timeframe_td - detail_data = self.detail_data[pair] - detail_data = detail_data.loc[ - (detail_data["date"] >= current_detail_time) - & (detail_data["date"] < exit_candle_end) - ].copy() - if len(detail_data) == 0: - # Fall back to "regular" data if no detail data was found for this candle - open_trade_count_start = self.backtest_loop( - row, pair, current_time, end_date, open_trade_count_start, trade_dir - ) - continue - detail_data.loc[:, "enter_long"] = row[LONG_IDX] - detail_data.loc[:, "exit_long"] = row[ELONG_IDX] - detail_data.loc[:, "enter_short"] = row[SHORT_IDX] - detail_data.loc[:, "exit_short"] = row[ESHORT_IDX] - detail_data.loc[:, "enter_tag"] = row[ENTER_TAG_IDX] - detail_data.loc[:, "exit_tag"] = row[EXIT_TAG_IDX] - is_first = True - current_time_det = current_time - for det_row in detail_data[HEADERS].values.tolist(): - self.dataprovider._set_dataframe_max_date(current_time_det) - open_trade_count_start = self.backtest_loop( - det_row, - pair, - current_time_det, - end_date, - open_trade_count_start, - trade_dir, - is_first, - ) - current_time_det += self.timeframe_detail_td - is_first = False - else: - self.dataprovider._set_dataframe_max_date(current_time) + detail_data = self.detail_data[pair] + detail_data = detail_data.loc[ + (detail_data["date"] >= current_detail_time) + & (detail_data["date"] < exit_candle_end) + ].copy() + if len(detail_data) == 0: + # Fall back to "regular" data if no detail data was found for this candle open_trade_count_start = self.backtest_loop( row, pair, current_time, end_date, open_trade_count_start, trade_dir ) - - # Move time one configured time_interval ahead. - self.progress.increment() - current_time += self.timeframe_td + continue + detail_data.loc[:, "enter_long"] = row[LONG_IDX] + detail_data.loc[:, "exit_long"] = row[ELONG_IDX] + detail_data.loc[:, "enter_short"] = row[SHORT_IDX] + detail_data.loc[:, "exit_short"] = row[ESHORT_IDX] + detail_data.loc[:, "enter_tag"] = row[ENTER_TAG_IDX] + detail_data.loc[:, "exit_tag"] = row[EXIT_TAG_IDX] + is_first = True + current_time_det = current_time + for det_row in detail_data[HEADERS].values.tolist(): + self.dataprovider._set_dataframe_max_date(current_time_det) + open_trade_count_start = self.backtest_loop( + det_row, + pair, + current_time_det, + end_date, + open_trade_count_start, + trade_dir, + is_first, + ) + current_time_det += self.timeframe_detail_td + is_first = False + else: + self.dataprovider._set_dataframe_max_date(current_time) + open_trade_count_start = self.backtest_loop( + row, pair, current_time, end_date, open_trade_count_start, trade_dir + ) self.handle_left_open(LocalTrade.bt_trades_open_pp, data=data) self.wallets.update() From b6f4e124ce4f249972eb6b849f0260457d9262d7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2024 11:36:54 +0200 Subject: [PATCH 02/11] chore: improve backtesting test details ensure all candles used the same pairlist ordering --- tests/optimize/test_backtesting.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index a7823c883..c2c6ce954 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument import random +from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta, timezone from pathlib import Path @@ -1485,6 +1486,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) default_conf["max_open_trades"] = 3 backtesting = Backtesting(default_conf) + vr_spy = mocker.spy(backtesting, "validate_row") backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.bot_loop_start = MagicMock() backtesting.strategy.advise_entry = _trend_alternate_hold # Override @@ -1503,6 +1505,17 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) # bot_loop_start is called once per candle. assert backtesting.strategy.bot_loop_start.call_count == 499 + # Validated row once per candle and pair + assert vr_spy.call_count == 2495 + # List of calls pair args - in batches of 5 (s) + calls_per_candle = defaultdict(list) + for call in vr_spy.call_args_list: + calls_per_candle[call[0][3]].append(call[0][1]) + + all_orients = [x for _, x in calls_per_candle.items()] + + assert all(x == ["ADA/BTC", "DASH/BTC", "ETH/BTC", "LTC/BTC", "NXT/BTC"] for x in all_orients) + # Make sure we have parallel trades assert len(evaluate_result_multi(results["results"], "5m", 2)) > 0 # make sure we don't have trades with more than configured max_open_trades From 7945eba38658b7e9e07f240b73fe779ba2a7279f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2024 11:51:36 +0200 Subject: [PATCH 03/11] feat: Evaluate pairs with open trades first This will enable further improved logic for pairs with no open trade. --- freqtrade/optimize/backtesting.py | 5 ++++- tests/optimize/test_backtesting.py | 21 ++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b26013a11..b6b7c113f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1400,7 +1400,10 @@ class Backtesting: ) while current_time <= end_date: is_first = True - for pair in pairs: + # Pairs that have open trades should be processed first + new_pairlist = list(dict.fromkeys([t.pair for t in LocalTrade.bt_trades_open] + pairs)) + + for pair in new_pairlist: yield current_time, pair, is_first is_first = False diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index c2c6ce954..9567a2cac 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1514,7 +1514,26 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) all_orients = [x for _, x in calls_per_candle.items()] - assert all(x == ["ADA/BTC", "DASH/BTC", "ETH/BTC", "LTC/BTC", "NXT/BTC"] for x in all_orients) + distinct_calls = [list(x) for x in set(tuple(x) for x in all_orients)] + + # All calls must be made for the full pairlist + assert all(len(x) == 5 for x in distinct_calls) + + # order varied - and is not always identical + assert not all( + x == ["ADA/BTC", "DASH/BTC", "ETH/BTC", "LTC/BTC", "NXT/BTC"] for x in distinct_calls + ) + # But some calls should've kept the original ordering + assert any( + x == ["ADA/BTC", "DASH/BTC", "ETH/BTC", "LTC/BTC", "NXT/BTC"] for x in distinct_calls + ) + assert ( + # Ordering can be different, but should be one of the following + any(x == ["ETH/BTC", "ADA/BTC", "DASH/BTC", "LTC/BTC", "NXT/BTC"] for x in distinct_calls) + or any( + x == ["ETH/BTC", "LTC/BTC", "ADA/BTC", "DASH/BTC", "NXT/BTC"] for x in distinct_calls + ) + ) # Make sure we have parallel trades assert len(evaluate_result_multi(results["results"], "5m", 2)) > 0 From 08c10c1f9b51ed162bd4fbc2c0f568e4f82f7811 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2024 12:34:51 +0200 Subject: [PATCH 04/11] chore: exclude right boundary from parallelism test --- freqtrade/data/btanalysis.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 2895b4181..a237b10f1 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -401,7 +401,15 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF timeframe_freq = timeframe_to_resample_freq(timeframe) dates = [ - pd.Series(pd.date_range(row[1]["open_date"], row[1]["close_date"], freq=timeframe_freq)) + pd.Series( + pd.date_range( + row[1]["open_date"], + row[1]["close_date"], + freq=timeframe_freq, + # Exclude right boundary - the date is the candle open date. + inclusive="left", + ) + ) for row in results[["open_date", "close_date"]].iterrows() ] deltas = [len(x) for x in dates] From 70f3018e67970556237d8df52c885623899edbc4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2024 13:19:53 +0200 Subject: [PATCH 05/11] feat: remove "open_trade_count_start" workaround Due to the updated pair ordering logic, we can open trades on different pairs during the same candle without superating the max_open_trades limit --- freqtrade/optimize/backtesting.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b6b7c113f..2edb65e1b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1332,10 +1332,9 @@ class Backtesting: pair: str, current_time: datetime, end_date: datetime, - open_trade_count_start: int, trade_dir: Optional[LongShort], is_first: bool = True, - ) -> int: + ) -> None: """ NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. @@ -1345,7 +1344,6 @@ class Backtesting: # 1. Manage currently open orders of active trades if self.manage_open_orders(t, current_time, row): # Close trade - open_trade_count_start -= 1 LocalTrade.remove_bt_trade(t) self.wallets.update() @@ -1361,13 +1359,9 @@ class Backtesting: and trade_dir is not None and not PairLocks.is_pair_locked(pair, row[DATE_IDX], trade_dir) ): - if self.trade_slot_available(open_trade_count_start): + if self.trade_slot_available(LocalTrade.bt_open_open_trade_count): trade = self._enter_trade(pair, row, trade_dir) if trade: - # TODO: hacky workaround to avoid opening > max_open_trades - # This emulates previous behavior - not sure if this is correct - # Prevents entering if the trade-slot was freed in this candle - open_trade_count_start += 1 self.wallets.update() else: self._collate_rejected(pair, row) @@ -1386,7 +1380,6 @@ class Backtesting: order = trade.select_order(trade.exit_side, is_open=True) if order: self._process_exit_order(order, trade, current_time, row, pair) - return open_trade_count_start def time_pair_generator( self, start_date: datetime, end_date: datetime, increment: timedelta, pairs: List[str] @@ -1439,7 +1432,6 @@ class Backtesting: start_date, end_date, self.timeframe_td, list(data.keys()) ): if is_first: - open_trade_count_start = LocalTrade.bt_open_open_trade_count self.check_abort() strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)( current_time=current_time @@ -1473,9 +1465,7 @@ class Backtesting: ].copy() if len(detail_data) == 0: # Fall back to "regular" data if no detail data was found for this candle - open_trade_count_start = self.backtest_loop( - row, pair, current_time, end_date, open_trade_count_start, trade_dir - ) + self.backtest_loop(row, pair, current_time, end_date, trade_dir) continue detail_data.loc[:, "enter_long"] = row[LONG_IDX] detail_data.loc[:, "exit_long"] = row[ELONG_IDX] @@ -1487,12 +1477,11 @@ class Backtesting: current_time_det = current_time for det_row in detail_data[HEADERS].values.tolist(): self.dataprovider._set_dataframe_max_date(current_time_det) - open_trade_count_start = self.backtest_loop( + self.backtest_loop( det_row, pair, current_time_det, end_date, - open_trade_count_start, trade_dir, is_first, ) @@ -1500,9 +1489,7 @@ class Backtesting: is_first = False else: self.dataprovider._set_dataframe_max_date(current_time) - open_trade_count_start = self.backtest_loop( - row, pair, current_time, end_date, open_trade_count_start, trade_dir - ) + self.backtest_loop(row, pair, current_time, end_date, trade_dir) self.handle_left_open(LocalTrade.bt_trades_open_pp, data=data) self.wallets.update() From 4882a18bf935b8539d7820a099d58321172a3fb0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2024 14:13:41 +0200 Subject: [PATCH 06/11] chore: add pair_detail test --- tests/optimize/test_backtesting.py | 133 ++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 1 deletion(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 9567a2cac..2bc4a8bce 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -16,7 +16,7 @@ from freqtrade.commands.optimize_commands import setup_optimize_configuration, s from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import BT_DATA_COLUMNS, evaluate_result_multi -from freqtrade.data.converter import clean_ohlcv_dataframe +from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_fill_up_missing_data from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange from freqtrade.enums import CandleType, ExitType, RunMode @@ -30,6 +30,7 @@ from freqtrade.util.datetime_helpers import dt_utc from tests.conftest import ( CURRENT_TEST_STRATEGY, EXMS, + generate_test_data, get_args, log_has, log_has_re, @@ -1560,6 +1561,136 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) assert len(evaluate_result_multi(results["results"], "5m", 1)) == 0 +@pytest.mark.parametrize("pair", ["ADA/USDT", "LTC/USDT"]) +@pytest.mark.parametrize("tres", [0, 20, 30]) +def test_backtest_multi_pair_detail( + default_conf_usdt, + fee, + mocker, + tres, + pair, +): + """ + literally the same as test_backtest_multi_pair - but with artificial data + and detail timeframe. + """ + + def _trend_alternate_hold(dataframe=None, metadata=None): + """ + Buy every xth candle - sell every other xth -2 (hold on to pairs a bit) + """ + if metadata["pair"] in ("ETH/USDT", "LTC/USDT"): + multi = 20 + else: + multi = 18 + dataframe["enter_long"] = np.where(dataframe.index % multi == 0, 1, 0) + dataframe["exit_long"] = np.where((dataframe.index + multi - 2) % multi == 0, 1, 0) + dataframe["enter_short"] = 0 + dataframe["exit_short"] = 0 + return dataframe + + default_conf_usdt["runmode"] = "backtest" + default_conf_usdt["stoploss"] = -1.0 + default_conf_usdt["minimal_roi"] = {"0": 100} + mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001) + mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float("inf")) + mocker.patch(f"{EXMS}.get_fee", fee) + patch_exchange(mocker) + + raw_candles_1m = generate_test_data("1m", 2500, "2022-01-03 12:00:00+00:00") + raw_candles = ohlcv_fill_up_missing_data(raw_candles_1m, "5m", "dummy") + + pairs = ["ADA/USDT", "DASH/USDT", "ETH/USDT", "LTC/USDT", "NXT/USDT"] + data = {pair: raw_candles for pair in pairs} + + # Only use 500 lines to increase performance + data = trim_dictlist(data, -500) + + # Remove data for one pair from the beginning of the data + if tres > 0: + data[pair] = data[pair][tres:].reset_index() + default_conf_usdt["timeframe"] = "5m" + default_conf_usdt["max_open_trades"] = 3 + + backtesting = Backtesting(default_conf_usdt) + vr_spy = mocker.spy(backtesting, "validate_row") + backtesting._set_strategy(backtesting.strategylist[0]) + backtesting.strategy.bot_loop_start = MagicMock() + backtesting.strategy.advise_entry = _trend_alternate_hold # Override + backtesting.strategy.advise_exit = _trend_alternate_hold # Override + + processed = backtesting.strategy.advise_all_indicators(data) + min_date, max_date = get_timerange(processed) + + backtest_conf = { + "processed": deepcopy(processed), + "start_date": min_date, + "end_date": max_date, + } + + results = backtesting.backtest(**backtest_conf) + + # bot_loop_start is called once per candle. + assert backtesting.strategy.bot_loop_start.call_count == 499 + # Validated row once per candle and pair + assert vr_spy.call_count == 2495 + # List of calls pair args - in batches of 5 (s) + calls_per_candle = defaultdict(list) + for call in vr_spy.call_args_list: + calls_per_candle[call[0][3]].append(call[0][1]) + + all_orients = [x for _, x in calls_per_candle.items()] + + distinct_calls = [list(x) for x in set(tuple(x) for x in all_orients)] + + # All calls must be made for the full pairlist + assert all(len(x) == 5 for x in distinct_calls) + + # order varied - and is not always identical + assert not all( + x == ["ADA/USDT", "DASH/USDT", "ETH/USDT", "LTC/USDT", "NXT/USDT"] for x in distinct_calls + ) + # But some calls should've kept the original ordering + assert any( + x == ["ADA/USDT", "DASH/USDT", "ETH/USDT", "LTC/USDT", "NXT/USDT"] for x in distinct_calls + ) + assert ( + # Ordering can be different, but should be one of the following + any( + x == ["ETH/USDT", "ADA/USDT", "DASH/USDT", "LTC/USDT", "NXT/USDT"] + for x in distinct_calls + ) + or any( + x == ["ETH/USDT", "LTC/USDT", "ADA/USDT", "DASH/USDT", "NXT/USDT"] + for x in distinct_calls + ) + ) + + # Make sure we have parallel trades + assert len(evaluate_result_multi(results["results"], "5m", 2)) > 0 + # make sure we don't have trades with more than configured max_open_trades + assert len(evaluate_result_multi(results["results"], "5m", 3)) == 0 + + # Cached data correctly removed amounts + offset = 1 if tres == 0 else 0 + removed_candles = len(data[pair]) - offset + assert len(backtesting.dataprovider.get_analyzed_dataframe(pair, "5m")[0]) == removed_candles + assert ( + len(backtesting.dataprovider.get_analyzed_dataframe("NXT/USDT", "5m")[0]) + == len(data["NXT/USDT"]) - 1 + ) + + backtesting.strategy.max_open_trades = 1 + backtesting.config.update({"max_open_trades": 1}) + backtest_conf = { + "processed": deepcopy(processed), + "start_date": min_date, + "end_date": max_date, + } + results = backtesting.backtest(**backtest_conf) + assert len(evaluate_result_multi(results["results"], "5m", 1)) == 0 + + def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): patch_exchange(mocker) mocker.patch("freqtrade.optimize.backtesting.Backtesting.backtest") From 530226dbe82a1f8c0ff95456afea589144a1ce17 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2024 14:19:44 +0200 Subject: [PATCH 07/11] chore: Add "use_detail" to detail test --- tests/optimize/test_backtesting.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 2bc4a8bce..8c1bce97c 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1561,6 +1561,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) assert len(evaluate_result_multi(results["results"], "5m", 1)) == 0 +@pytest.mark.parametrize("use_detail", [True, False]) @pytest.mark.parametrize("pair", ["ADA/USDT", "LTC/USDT"]) @pytest.mark.parametrize("tres", [0, 20, 30]) def test_backtest_multi_pair_detail( @@ -1569,6 +1570,7 @@ def test_backtest_multi_pair_detail( mocker, tres, pair, + use_detail, ): """ literally the same as test_backtest_multi_pair - but with artificial data @@ -1592,6 +1594,10 @@ def test_backtest_multi_pair_detail( default_conf_usdt["runmode"] = "backtest" default_conf_usdt["stoploss"] = -1.0 default_conf_usdt["minimal_roi"] = {"0": 100} + + if use_detail: + default_conf_usdt["timeframe_detail"] = "1m" + mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001) mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float("inf")) mocker.patch(f"{EXMS}.get_fee", fee) @@ -1602,6 +1608,7 @@ def test_backtest_multi_pair_detail( pairs = ["ADA/USDT", "DASH/USDT", "ETH/USDT", "LTC/USDT", "NXT/USDT"] data = {pair: raw_candles for pair in pairs} + detail_data = {pair: raw_candles_1m for pair in pairs} # Only use 500 lines to increase performance data = trim_dictlist(data, -500) @@ -1614,6 +1621,8 @@ def test_backtest_multi_pair_detail( backtesting = Backtesting(default_conf_usdt) vr_spy = mocker.spy(backtesting, "validate_row") + bl_spy = mocker.spy(backtesting, "backtest_loop") + backtesting.detail_data = detail_data backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.bot_loop_start = MagicMock() backtesting.strategy.advise_entry = _trend_alternate_hold # Override @@ -1634,6 +1643,15 @@ def test_backtest_multi_pair_detail( assert backtesting.strategy.bot_loop_start.call_count == 499 # Validated row once per candle and pair assert vr_spy.call_count == 2495 + + if use_detail: + # Backtest loop is called once per candle per pair + # Exact numbers depend on trade state - but should be around 3_800 + assert bl_spy.call_count > 3_800 + assert bl_spy.call_count < 3_900 + else: + assert bl_spy.call_count < 2495 + # List of calls pair args - in batches of 5 (s) calls_per_candle = defaultdict(list) for call in vr_spy.call_args_list: From 5773d1fd8d95c5eace2f2cb1d57ed1a80b0a7f99 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2024 14:40:19 +0200 Subject: [PATCH 08/11] docs: Update documentation for new flow --- docs/backtesting.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 5adeae54b..00a79592f 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -530,10 +530,9 @@ You can then load the trades to perform further analysis as shown in the [data a Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions: - Exchange [trading limits](#trading-limits-in-backtesting) are respected -- Entries happen at open-price +- Entries happen at open-price unless a custom price logic has been specified - All orders are filled at the requested price (no slippage) as long as the price is within the candle's high/low range - Exit-signal exits happen at open-price of the consecutive candle -- Exits don't free their trade slot for a new trade until the next candle - Exit-signal is favored over Stoploss, because exit-signals are assumed to trigger on candle's open - ROI - Exits are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the exit will be at 2%) From b727e5ca1c2628f7a99e380516d396d907464dc0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2024 14:43:54 +0200 Subject: [PATCH 09/11] chore: simplify update code --- tests/optimize/test_backtesting.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 8c1bce97c..c396e1135 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1591,9 +1591,13 @@ def test_backtest_multi_pair_detail( dataframe["exit_short"] = 0 return dataframe - default_conf_usdt["runmode"] = "backtest" - default_conf_usdt["stoploss"] = -1.0 - default_conf_usdt["minimal_roi"] = {"0": 100} + default_conf_usdt.update( + { + "runmode": "backtest", + "stoploss": -1.0, + "minimal_roi": {"0": 100}, + } + ) if use_detail: default_conf_usdt["timeframe_detail"] = "1m" From 50835c878e7ea50ded65f9aed2bf7ede9a1780fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2024 15:07:42 +0200 Subject: [PATCH 10/11] chore: add more test coverage --- tests/optimize/test_backtesting.py | 138 ++++++++++++++++++++++------- 1 file changed, 106 insertions(+), 32 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index c396e1135..5bc113a62 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1656,38 +1656,6 @@ def test_backtest_multi_pair_detail( else: assert bl_spy.call_count < 2495 - # List of calls pair args - in batches of 5 (s) - calls_per_candle = defaultdict(list) - for call in vr_spy.call_args_list: - calls_per_candle[call[0][3]].append(call[0][1]) - - all_orients = [x for _, x in calls_per_candle.items()] - - distinct_calls = [list(x) for x in set(tuple(x) for x in all_orients)] - - # All calls must be made for the full pairlist - assert all(len(x) == 5 for x in distinct_calls) - - # order varied - and is not always identical - assert not all( - x == ["ADA/USDT", "DASH/USDT", "ETH/USDT", "LTC/USDT", "NXT/USDT"] for x in distinct_calls - ) - # But some calls should've kept the original ordering - assert any( - x == ["ADA/USDT", "DASH/USDT", "ETH/USDT", "LTC/USDT", "NXT/USDT"] for x in distinct_calls - ) - assert ( - # Ordering can be different, but should be one of the following - any( - x == ["ETH/USDT", "ADA/USDT", "DASH/USDT", "LTC/USDT", "NXT/USDT"] - for x in distinct_calls - ) - or any( - x == ["ETH/USDT", "LTC/USDT", "ADA/USDT", "DASH/USDT", "NXT/USDT"] - for x in distinct_calls - ) - ) - # Make sure we have parallel trades assert len(evaluate_result_multi(results["results"], "5m", 2)) > 0 # make sure we don't have trades with more than configured max_open_trades @@ -1713,6 +1681,112 @@ def test_backtest_multi_pair_detail( assert len(evaluate_result_multi(results["results"], "5m", 1)) == 0 +@pytest.mark.parametrize("use_detail", [True, False]) +def test_backtest_multi_pair_long_short_switch( + default_conf_usdt, + fee, + mocker, + use_detail, +): + """ + literally the same as test_backtest_multi_pair - but with artificial data + and detail timeframe. + """ + + def _trend_alternate_hold(dataframe=None, metadata=None): + """ + Buy every xth candle - sell every other xth -2 (hold on to pairs a bit) + """ + if metadata["pair"] in ("ETH/USDT", "LTC/USDT"): + multi = 20 + else: + multi = 18 + dataframe["enter_long"] = np.where(dataframe.index % multi == 0, 1, 0) + dataframe["exit_long"] = np.where((dataframe.index + multi - 2) % multi == 0, 1, 0) + dataframe["enter_short"] = dataframe["exit_long"] + dataframe["exit_short"] = dataframe["enter_long"] + return dataframe + + default_conf_usdt.update( + { + "runmode": "backtest", + "timeframe": "5m", + "max_open_trades": 1, + "stoploss": -1.0, + "minimal_roi": {"0": 100}, + "margin_mode": "isolated", + "trading_mode": "futures", + } + ) + + if use_detail: + default_conf_usdt["timeframe_detail"] = "1m" + + mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001) + mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float("inf")) + mocker.patch(f"{EXMS}.get_fee", fee) + patch_exchange(mocker) + + raw_candles_1m = generate_test_data("1m", 2500, "2022-01-03 12:00:00+00:00") + raw_candles = ohlcv_fill_up_missing_data(raw_candles_1m, "5m", "dummy") + + pairs = [ + "ETH/USDT:USDT", + ] + default_conf_usdt["exchange"]["pair_whitelist"] = pairs + # Fake whitelist to avoid some mock data issues + mocker.patch(f"{EXMS}.get_maintenance_ratio_and_amt", return_value=(0.01, 0.01)) + + data = {pair: raw_candles for pair in pairs} + detail_data = {pair: raw_candles_1m for pair in pairs} + + # Only use 500 lines to increase performance + data = trim_dictlist(data, -500) + + backtesting = Backtesting(default_conf_usdt) + vr_spy = mocker.spy(backtesting, "validate_row") + bl_spy = mocker.spy(backtesting, "backtest_loop") + backtesting.detail_data = detail_data + backtesting.funding_fee_timeframe_secs = 3600 * 8 # 8h + backtesting.futures_data = {pair: pd.DataFrame() for pair in pairs} + + backtesting.strategylist[0].can_short = True + backtesting._set_strategy(backtesting.strategylist[0]) + backtesting.strategy.bot_loop_start = MagicMock() + backtesting.strategy.advise_entry = _trend_alternate_hold # Override + backtesting.strategy.advise_exit = _trend_alternate_hold # Override + + processed = backtesting.strategy.advise_all_indicators(data) + min_date, max_date = get_timerange(processed) + + backtest_conf = { + "processed": deepcopy(processed), + "start_date": min_date, + "end_date": max_date, + } + + results = backtesting.backtest(**backtest_conf) + + # bot_loop_start is called once per candle. + assert backtesting.strategy.bot_loop_start.call_count == 499 + # Validated row once per candle and pair + assert vr_spy.call_count == 499 + + if use_detail: + # Backtest loop is called once per candle per pair + assert bl_spy.call_count == 1071 + else: + assert bl_spy.call_count == 479 + + # Make sure we have parallel trades + assert len(evaluate_result_multi(results["results"], "5m", 0)) > 0 + # make sure we don't have trades with more than configured max_open_trades + assert len(evaluate_result_multi(results["results"], "5m", 1)) == 0 + + # Expect 26 results initially + assert len(results["results"]) == 30 + + def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): patch_exchange(mocker) mocker.patch("freqtrade.optimize.backtesting.Backtesting.backtest") From 948e67a2b741e1563a7e5392d489bf65196944e2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Aug 2024 07:14:08 +0200 Subject: [PATCH 11/11] docs: improved wording --- docs/backtesting.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backtesting.md b/docs/backtesting.md index 00a79592f..2feba7ada 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -533,6 +533,7 @@ Since backtesting lacks some detailed information about what happens within a ca - Entries happen at open-price unless a custom price logic has been specified - All orders are filled at the requested price (no slippage) as long as the price is within the candle's high/low range - Exit-signal exits happen at open-price of the consecutive candle +- Exits free their trade slot for a new trade with a different pair - Exit-signal is favored over Stoploss, because exit-signals are assumed to trigger on candle's open - ROI - Exits are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the exit will be at 2%)