From 5fffc5033a9f69daa54719a55956260908d89ba7 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Tue, 18 Jan 2022 11:00:51 +0200 Subject: [PATCH 1/6] Rework backtesting --no-cahche to --cache=[none, day, week, month]. Fix an issue where config modification during runtime would prevent use of cached results. --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 10 ++-- freqtrade/configuration/configuration.py | 4 +- freqtrade/constants.py | 3 ++ freqtrade/data/btanalysis.py | 48 +++++++++++++------ freqtrade/optimize/backtesting.py | 30 ++++++++---- freqtrade/optimize/optimize_reports.py | 3 +- tests/optimize/test_backtesting.py | 59 +++++++++++++++++++----- 8 files changed, 117 insertions(+), 42 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 119a45662..290865a04 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -24,7 +24,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", "enable_protections", "dry_run_wallet", "timeframe_detail", "strategy_list", "export", "exportfilename", - "backtest_breakdown", "no_backtest_cache"] + "backtest_breakdown", "backtest_cache"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "position_stacking", "use_max_market_positions", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 0fb93f0b8..87266fd68 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -205,10 +205,12 @@ AVAILABLE_CLI_OPTIONS = { nargs='+', choices=constants.BACKTEST_BREAKDOWNS ), - "no_backtest_cache": Arg( - '--no-cache', - help='Do not reuse cached backtest results.', - action='store_true' + "backtest_cache": Arg( + '--cache', + help='Load a cached backtest result no older than specified age.', + metavar='AGE', + default=constants.BACKTEST_CACHE_DEFAULT, + choices=constants.BACKTEST_CACHE_AGE, ), # Edge "stoploss_range": Arg( diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 066097916..3ac2e3ddd 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -276,8 +276,8 @@ class Configuration: self._args_to_config(config, argname='backtest_breakdown', logstring='Parameter --breakdown detected ...') - self._args_to_config(config, argname='no_backtest_cache', - logstring='Parameter --no-cache detected ...') + self._args_to_config(config, argname='backtest_cache', + logstring='Parameter --cache={} detected ...') self._args_to_config(config, argname='disableparamexport', logstring='Parameter --disableparamexport detected: {} ...') diff --git a/freqtrade/constants.py b/freqtrade/constants.py index f15759ea5..a06666695 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -34,6 +34,9 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] BACKTEST_BREAKDOWNS = ['day', 'week', 'month'] +BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month'] +BACKTEST_CACHE_DEFAULT = 'day' +BACKTEST_CACHE_NONE = 'none' DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 27ce8e0ba..1a4d2b1d1 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -3,6 +3,7 @@ Helpers when analyzing backtest data """ import logging from copy import copy +from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union @@ -143,12 +144,24 @@ def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]: return data -def find_existing_backtest_stats(dirname: Union[Path, str], - run_ids: Dict[str, str]) -> Dict[str, Any]: +def _load_and_merge_backtest_result(strategy_name: str, filename: Path, results: Dict[str, Any]): + bt_data = load_backtest_stats(filename) + for k in ('metadata', 'strategy'): + results[k][strategy_name] = bt_data[k][strategy_name] + comparison = bt_data['strategy_comparison'] + for i in range(len(comparison)): + if comparison[i]['key'] == strategy_name: + results['strategy_comparison'].append(comparison[i]) + break + + +def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, str], + min_backtest_date: datetime = None) -> Dict[str, Any]: """ Find existing backtest stats that match specified run IDs and load them. :param dirname: pathlib.Path object, or string pointing to the file. :param run_ids: {strategy_name: id_string} dictionary. + :param min_backtest_date: do not load a backtest older than specified date. :return: results dict. """ # Copy so we can modify this dict without affecting parent scope. @@ -169,18 +182,27 @@ def find_existing_backtest_stats(dirname: Union[Path, str], break for strategy_name, run_id in list(run_ids.items()): - if metadata.get(strategy_name, {}).get('run_id') == run_id: - # TODO: load_backtest_stats() may load an old version of backtest which is - # incompatible with current version. + strategy_metadata = metadata.get(strategy_name, None) + if not strategy_metadata: + # This strategy is not present in analyzed backtest. + continue + + if min_backtest_date is not None: + try: + backtest_date = strategy_metadata['backtest_start_time'] + except KeyError: + # Older metadata format without backtest time, too old to consider. + return results + backtest_date = datetime.fromtimestamp(backtest_date, tz=timezone.utc) + if backtest_date < min_backtest_date: + # Do not use a cached result for this strategy as first result is too old. + del run_ids[strategy_name] + continue + + if strategy_metadata['run_id'] == run_id: del run_ids[strategy_name] - bt_data = load_backtest_stats(filename) - for k in ('metadata', 'strategy'): - results[k][strategy_name] = bt_data[k][strategy_name] - comparison = bt_data['strategy_comparison'] - for i in range(len(comparison)): - if comparison[i]['key'] == strategy_name: - results['strategy_comparison'].append(comparison[i]) - break + _load_and_merge_backtest_result(strategy_name, filename, results) + if len(run_ids) == 0: break return results diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a4a5fd140..b98ea1999 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame +from freqtrade import constants from freqtrade.configuration import TimeRange, validate_config_consistency from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data import history @@ -64,6 +65,7 @@ class Backtesting: self.results: Dict[str, Any] = {} config['dry_run'] = True + self.run_ids: Dict[str, str] = {} self.strategylist: List[IStrategy] = [] self.all_results: Dict[str, Dict] = {} @@ -728,7 +730,7 @@ class Backtesting: ) backtest_end_time = datetime.now(timezone.utc) results.update({ - 'run_id': get_strategy_run_id(strat), + 'run_id': self.run_ids.get(strat.get_strategy_name(), ''), 'backtest_start_time': int(backtest_start_time.timestamp()), 'backtest_end_time': int(backtest_end_time.timestamp()), }) @@ -736,6 +738,20 @@ class Backtesting: return min_date, max_date + def _get_min_cached_backtest_date(self): + min_backtest_date = None + backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT) + if self.timerange.stopts == 0 or datetime.fromtimestamp( + self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc): + logger.warning('Backtest result caching disabled due to use of open-ended timerange.') + elif backtest_cache_age == 'day': + min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(days=1) + elif backtest_cache_age == 'week': + min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=1) + elif backtest_cache_age == 'month': + min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=4) + return min_backtest_date + def start(self) -> None: """ Run backtesting end-to-end @@ -747,21 +763,17 @@ class Backtesting: self.load_bt_data_detail() logger.info("Dataload complete. Calculating indicators") - run_ids = { + self.run_ids = { strategy.get_strategy_name(): get_strategy_run_id(strategy) for strategy in self.strategylist } # Load previous result that will be updated incrementally. # This can be circumvented in certain instances in combination with downloading more data - if self.timerange.stopts == 0 or datetime.fromtimestamp( - self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc): - self.config['no_backtest_cache'] = True - logger.warning('Backtest result caching disabled due to use of open-ended timerange.') - - if not self.config.get('no_backtest_cache', False): + min_backtest_date = self._get_min_cached_backtest_date() + if min_backtest_date is not None: self.results = find_existing_backtest_stats( - self.config['user_data_dir'] / 'backtest_results', run_ids) + self.config['user_data_dir'] / 'backtest_results', self.run_ids, min_backtest_date) for strat in self.strategylist: if self.results and strat.get_strategy_name() in self.results['strategy']: diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 46930d7b1..859238af3 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -527,7 +527,8 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], strat_stats = generate_strategy_stats(pairlist, strategy, content, min_date, max_date, market_change=market_change) metadata[strategy] = { - 'run_id': content['run_id'] + 'run_id': content['run_id'], + 'backtest_start_time': content['backtest_start_time'], } result['strategy'][strategy] = strat_stats diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 7dd0abd4a..edd0faba3 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -11,6 +11,7 @@ import pandas as pd import pytest from arrow import Arrow +from freqtrade import constants from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting from freqtrade.configuration import TimeRange from freqtrade.data import history @@ -1242,8 +1243,11 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker, @pytest.mark.filterwarnings("ignore:deprecated") -def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testdatadir): - +@pytest.mark.parametrize('run_id', ['2', 'changed']) +@pytest.mark.parametrize('start_delta', [{'days': 0}, {'days': 1}, {'weeks': 1}, {'weeks': 4}]) +@pytest.mark.parametrize('cache', constants.BACKTEST_CACHE_AGE) +def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testdatadir, run_id, + start_delta, cache): default_conf.update({ "use_sell_signal": True, "sell_profit_only": False, @@ -1263,9 +1267,19 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) mocker.patch('freqtrade.optimize.backtesting.show_backtest_results', MagicMock()) + now = min_backtest_date = datetime.now(tz=timezone.utc) + start_time = now - timedelta(**start_delta) + timedelta(hours=1) + if cache == 'none': + min_backtest_date = now + timedelta(days=1) + elif cache == 'day': + min_backtest_date = now - timedelta(days=1) + elif cache == 'week': + min_backtest_date = now - timedelta(weeks=1) + elif cache == 'month': + min_backtest_date = now - timedelta(weeks=4) load_backtest_metadata = MagicMock(return_value={ - 'StrategyTestV2': {'run_id': '1'}, - 'TestStrategyLegacyV1': {'run_id': 'changed'} + 'StrategyTestV2': {'run_id': '1', 'backtest_start_time': now.timestamp()}, + 'TestStrategyLegacyV1': {'run_id': run_id, 'backtest_start_time': start_time.timestamp()} }) load_backtest_stats = MagicMock(side_effect=[ { @@ -1279,7 +1293,8 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda 'strategy_comparison': [{'key': 'TestStrategyLegacyV1'}] } ]) - mocker.patch('pathlib.Path.glob', return_value=['not important']) + mocker.patch('pathlib.Path.glob', return_value=[ + Path(datetime.strftime(datetime.now(), 'backtest-result-%Y-%m-%d_%H-%M-%S.json'))]) mocker.patch.multiple('freqtrade.data.btanalysis', load_backtest_metadata=load_backtest_metadata, load_backtest_stats=load_backtest_stats) @@ -1296,29 +1311,49 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda '--timerange', '1510694220-1510700340', '--enable-position-stacking', '--disable-max-market-positions', + '--cache', cache, '--strategy-list', 'StrategyTestV2', 'TestStrategyLegacyV1', ] args = get_args(args) start_backtesting(args) - # 1 backtest, 1 loaded from cache - assert backtestmock.call_count == 1 # check the logs, that will contain the backtest result exists = [ 'Parameter -i/--timeframe detected ... Using timeframe: 1m ...', - 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Loading data from 2017-11-14 20:57:00 ' 'up to 2017-11-14 22:58:00 (0 days).', - 'Backtesting with data from 2017-11-14 21:17:00 ' - 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...', - 'Reusing result of previous backtest for StrategyTestV2', - 'Running backtesting for Strategy TestStrategyLegacyV1', ] for line in exists: assert log_has(line, caplog) + + if cache == 'none': + assert backtestmock.call_count == 2 + exists = [ + 'Running backtesting for Strategy StrategyTestV2', + 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', + 'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).', + ] + elif run_id == '2' and min_backtest_date < start_time: + assert backtestmock.call_count == 0 + exists = [ + 'Reusing result of previous backtest for StrategyTestV2', + 'Reusing result of previous backtest for TestStrategyLegacyV1', + ] + else: + exists = [ + 'Reusing result of previous backtest for StrategyTestV2', + 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', + 'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).', + ] + assert backtestmock.call_count == 1 + + for line in exists: + assert log_has(line, caplog) From d319204deaf6043584d247c3ee36dbf46fdf22db Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Jan 2022 20:08:09 +0100 Subject: [PATCH 2/6] Add note about legacy metadata format --- freqtrade/constants.py | 1 - freqtrade/data/btanalysis.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index a06666695..504c7dce9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -36,7 +36,6 @@ AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] BACKTEST_BREAKDOWNS = ['day', 'week', 'month'] BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month'] BACKTEST_CACHE_DEFAULT = 'day' -BACKTEST_CACHE_NONE = 'none' DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 1a4d2b1d1..57eec6eea 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -191,6 +191,9 @@ def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, s try: backtest_date = strategy_metadata['backtest_start_time'] except KeyError: + # TODO: this can be removed starting from feb 2022 + # The metadata-file without start_time was only available in develop + # and was never included in an official release. # Older metadata format without backtest time, too old to consider. return results backtest_date = datetime.fromtimestamp(backtest_date, tz=timezone.utc) From afe46a55f7acc196b7c4b4fa644a53f6a14e4ddf Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Jan 2022 20:19:17 +0100 Subject: [PATCH 3/6] Add documentation for --cache backtest option --- docs/backtesting.md | 9 ++++++--- freqtrade/commands/cli_options.py | 3 +-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index eae6ac4a9..7420c1dec 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -22,6 +22,7 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export {none,trades}] [--export-filename PATH] [--breakdown {day,week,month} [{day,week,month} ...]] + [--cache {none,day,week,month}] optional arguments: -h, --help show this help message and exit @@ -76,7 +77,9 @@ optional arguments: _today.json` --breakdown {day,week,month} [{day,week,month} ...] Show backtesting breakdown per [day, week, month]. - --no-cache Do not reuse cached backtest results. + --cache {none,day,week,month} + Load a cached backtest result no older than specified + age (default: day). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -460,11 +463,11 @@ The output will show a table containing the realized absolute Profit (in stake c ### Backtest result caching -To save time, by default backtest will reuse a cached result when backtested strategy and config match that of previous backtest. To force a new backtest despite existing result for identical run specify `--no-cache` parameter. +To save time, by default backtest will reuse a cached result from within the last day when the backtested strategy and config match that of a previous backtest. To force a new backtest despite existing result for an identical run specify `--cache none` parameter. !!! Warning Caching is automatically disabled for open-ended timeranges (`--timerange 20210101-`), as freqtrade cannot ensure reliably that the underlying data didn't change. It can also use cached results where it shouldn't if the original backtest had missing data at the end, which was fixed by downloading more data. - In this instance, please use `--no-cache` once to get a fresh backtest. + In this instance, please use `--cache none` once to force a fresh backtest. ### Further backtest-result analysis diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 87266fd68..fc5542c52 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -207,8 +207,7 @@ AVAILABLE_CLI_OPTIONS = { ), "backtest_cache": Arg( '--cache', - help='Load a cached backtest result no older than specified age.', - metavar='AGE', + help='Load a cached backtest result no older than specified age (default: %(default)s).', default=constants.BACKTEST_CACHE_DEFAULT, choices=constants.BACKTEST_CACHE_AGE, ), From e9baabce6f326de728f2e795a7cb5eca432ffc7e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Jan 2022 08:22:25 +0100 Subject: [PATCH 4/6] Store results when backtesting via API --- freqtrade/rpc/api_server/api_backtest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index d110134d7..8101f54f5 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -39,7 +39,8 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac # Start backtesting # Initialize backtesting object def run_backtest(): - from freqtrade.optimize.optimize_reports import generate_backtest_stats + from freqtrade.optimize.optimize_reports import (generate_backtest_stats, + store_backtest_stats) from freqtrade.resolvers import StrategyResolver asyncio.set_event_loop(asyncio.new_event_loop()) try: @@ -83,6 +84,10 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac ApiServer._bt.results = generate_backtest_stats( ApiServer._bt_data, ApiServer._bt.all_results, min_date=min_date, max_date=max_date) + + if btconfig.get('export', 'none') == 'trades': + store_backtest_stats(btconfig['exportfilename'], ApiServer._bt.results) + logger.info("Backtest finished.") except DependencyException as e: From a6c7f4554539ec1490bf731fbcb30548c3510d3b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Jan 2022 06:44:41 +0100 Subject: [PATCH 5/6] Update webserver backtseting to reuse prior results --- freqtrade/optimize/backtesting.py | 25 +++++++++++++----------- freqtrade/rpc/api_server/api_backtest.py | 20 +++++++++++++------ 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b98ea1999..ae4001f5f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -752,17 +752,7 @@ class Backtesting: min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=4) return min_backtest_date - def start(self) -> None: - """ - Run backtesting end-to-end - :return: None - """ - data: Dict[str, Any] = {} - - data, timerange = self.load_bt_data() - self.load_bt_data_detail() - logger.info("Dataload complete. Calculating indicators") - + def load_prior_backtest(self): self.run_ids = { strategy.get_strategy_name(): get_strategy_run_id(strategy) for strategy in self.strategylist @@ -775,6 +765,19 @@ class Backtesting: self.results = find_existing_backtest_stats( self.config['user_data_dir'] / 'backtest_results', self.run_ids, min_backtest_date) + def start(self) -> None: + """ + Run backtesting end-to-end + :return: None + """ + data: Dict[str, Any] = {} + + data, timerange = self.load_bt_data() + self.load_bt_data_detail() + logger.info("Dataload complete. Calculating indicators") + + self.load_prior_backtest() + for strat in self.strategylist: if self.results and strat.get_strategy_name() in self.results['strategy']: # When previous result hash matches - reuse that result and skip backtesting. diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 8101f54f5..97b7b7989 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -77,13 +77,21 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac lastconfig['enable_protections'] = btconfig.get('enable_protections') lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') - ApiServer._bt.abort = False - min_date, max_date = ApiServer._bt.backtest_one_strategy( - strat, ApiServer._bt_data, ApiServer._bt_timerange) + ApiServer._bt.results = {} + ApiServer._bt.load_prior_backtest() - ApiServer._bt.results = generate_backtest_stats( - ApiServer._bt_data, ApiServer._bt.all_results, - min_date=min_date, max_date=max_date) + ApiServer._bt.abort = False + if (ApiServer._bt.results and + strat.get_strategy_name() in ApiServer._bt.results['strategy']): + # When previous result hash matches - reuse that result and skip backtesting. + logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}') + else: + min_date, max_date = ApiServer._bt.backtest_one_strategy( + strat, ApiServer._bt_data, ApiServer._bt_timerange) + + ApiServer._bt.results = generate_backtest_stats( + ApiServer._bt_data, ApiServer._bt.all_results, + min_date=min_date, max_date=max_date) if btconfig.get('export', 'none') == 'trades': store_backtest_stats(btconfig['exportfilename'], ApiServer._bt.results) From d5499058562370ff62d301f13ec16c0e1458444b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Jan 2022 07:11:48 +0100 Subject: [PATCH 6/6] Api-backtest to test new functionality --- tests/rpc/test_rpc_apiserver.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index d6a58322a..207d80cef 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1326,7 +1326,7 @@ def test_sysinfo(botclient): assert 'ram_pct' in result -def test_api_backtesting(botclient, mocker, fee, caplog): +def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): ftbot, client = botclient mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) @@ -1347,6 +1347,11 @@ def test_api_backtesting(botclient, mocker, fee, caplog): assert result['status'] == 'reset' assert not result['running'] assert result['status_msg'] == 'Backtest reset' + ftbot.config['export'] = 'trades' + ftbot.config['backtest_cache'] = 'none' + ftbot.config['user_data_dir'] = Path(tmpdir) + ftbot.config['exportfilename'] = Path(tmpdir) / "backtest_results" + ftbot.config['exportfilename'].mkdir() # start backtesting data = { @@ -1421,6 +1426,14 @@ def test_api_backtesting(botclient, mocker, fee, caplog): rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) assert log_has("Backtesting caused an error: ", caplog) + ftbot.config['backtest_cache'] = 'day' + + # Rerun backtest (should get previous result) + rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) + assert_response(rc) + result = rc.json() + assert log_has_re('Reusing result of previous backtest.*', caplog) + # Delete backtesting to avoid leakage since the backtest-object may stick around. rc = client_delete(client, f"{BASE_URI}/backtest") assert_response(rc)