From 162e94455b3cf928ac68bd895a7674f1dbe0ce8b Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 16 Mar 2022 12:16:24 +0000 Subject: [PATCH 01/23] Add support for storing buy candle indicator rows in backtesting results --- freqtrade/optimize/backtesting.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 79c861ee8..1a8c0903c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -64,7 +64,8 @@ class Backtesting: config['dry_run'] = True self.strategylist: List[IStrategy] = [] self.all_results: Dict[str, Dict] = {} - + self.processed_dfs: Dict[str, Dict] = {} + self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) self.dataprovider = DataProvider(self.config, None) @@ -136,6 +137,10 @@ class Backtesting: self.config['startup_candle_count'] = self.required_startup self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) + self.enable_backtest_signal_candle_export = False + if self.config.get('enable_backtest_signal_candle_export', None) is not None: + self.enable_backtest_signal_candle_export = bool(self.config.get('enable_backtest_signal_candle_export')) + self.progress = BTProgress() self.abort = False @@ -636,6 +641,25 @@ class Backtesting: }) self.all_results[self.strategy.get_strategy_name()] = results + if self.enable_backtest_signal_candle_export: + signal_candles_only = {} + for pair in preprocessed_tmp.keys(): + signal_candles_only_df = DataFrame() + + pairdf = preprocessed_tmp[pair] + resdf = results['results'] + pairresults = resdf.loc[(resdf["pair"] == pair)] + + if pairdf.shape[0] > 0: + for t, v in pairresults.open_date.items(): + allinds = pairdf.loc[(pairdf['date'] < v)] + signal_inds = allinds.iloc[[-1]] + signal_candles_only_df = signal_candles_only_df.append(signal_inds) + + signal_candles_only[pair] = signal_candles_only_df + + self.processed_dfs[self.strategy.get_strategy_name()] = signal_candles_only + return min_date, max_date def start(self) -> None: From 26ba899d7d565581763d7509e5a041ad4bcb7160 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 16 Apr 2022 14:37:36 +0100 Subject: [PATCH 02/23] Add constant, boolean check, rename option to fit with other x_enable, check that RunMode is BACKTEST --- freqtrade/constants.py | 1 + freqtrade/optimize/backtesting.py | 10 ++++------ freqtrade/optimize/optimize_reports.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c6a2ab5d3..d21020a3f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -380,6 +380,7 @@ CONF_SCHEMA = { }, 'position_adjustment_enable': {'type': 'boolean'}, 'max_entry_position_adjustment': {'type': ['integer', 'number'], 'minimum': -1}, + 'backtest_signal_candle_export_enable': {'type': 'boolean'}, }, 'definitions': { 'exchange': { diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 11704a70b..b8f63d006 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -19,7 +19,7 @@ from freqtrade.data import history from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider -from freqtrade.enums import BacktestState, CandleType, ExitCheckTuple, ExitType, TradingMode +from freqtrade.enums import BacktestState, CandleType, ExitCheckTuple, ExitType, TradingMode, RunMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import get_strategy_run_id @@ -74,7 +74,7 @@ class Backtesting: self.strategylist: List[IStrategy] = [] self.all_results: Dict[str, Dict] = {} self.processed_dfs: Dict[str, Dict] = {} - + self._exchange_name = self.config['exchange']['name'] self.exchange = ExchangeResolver.load_exchange(self._exchange_name, self.config) self.dataprovider = DataProvider(self.config, self.exchange) @@ -129,9 +129,7 @@ class Backtesting: self.config['startup_candle_count'] = self.required_startup self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) - self.enable_backtest_signal_candle_export = False - if self.config.get('enable_backtest_signal_candle_export', None) is not None: - self.enable_backtest_signal_candle_export = bool(self.config.get('enable_backtest_signal_candle_export')) + self.backtest_signal_candle_export_enable = self.config.get('backtest_signal_candle_export_enable', False) self.trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) # strategies which define "can_short=True" will fail to load in Spot mode. @@ -1076,7 +1074,7 @@ class Backtesting: }) self.all_results[self.strategy.get_strategy_name()] = results - if self.enable_backtest_signal_candle_export: + if self.backtest_signal_candle_export_enable and self.dataprovider.runmode == RunMode.BACKTEST: signal_candles_only = {} for pair in preprocessed_tmp.keys(): signal_candles_only_df = DataFrame() diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index c08fa07a1..a97a6cf0f 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -44,6 +44,25 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN) file_dump_json(latest_filename, {'latest_backtest': str(filename.name)}) +def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> None: + """ + Stores backtest trade signal candles + :param recordfilename: Path object, which can either be a filename or a directory. + Filenames will be appended with a timestamp right before the suffix + while for directories, /backtest-result-_signals.pkl will be used as filename + :param stats: Dict containing the backtesting signal candles + """ + if recordfilename.is_dir(): + filename = (recordfilename / + f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl') + else: + filename = Path.joinpath( + recordfilename.parent, + f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' + ).with_suffix(recordfilename.suffix) + + with open(filename, 'wb') as f: + pickle.dump(candles, f) def _get_line_floatfmt(stake_currency: str) -> List[str]: """ From 21734c5de77697cb75a5ebee8e52b1b9bce6b10a Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 16 Apr 2022 14:46:30 +0100 Subject: [PATCH 03/23] Add pickle import --- freqtrade/optimize/backtesting.py | 1 + freqtrade/optimize/optimize_reports.py | 1 + 2 files changed, 2 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b8f63d006..cc7dbb1fb 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -8,6 +8,7 @@ from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple +import pickle from numpy import nan from pandas import DataFrame diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index a97a6cf0f..4265ca70d 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -3,6 +3,7 @@ from copy import deepcopy from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Dict, List, Union +import pickle from numpy import int64 from pandas import DataFrame, to_datetime From 8990ba27099357c7f63540fd05e7c7db98961270 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 16 Apr 2022 14:49:53 +0100 Subject: [PATCH 04/23] Fix store signal candles --- freqtrade/optimize/backtesting.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cc7dbb1fb..4ed3f85be 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -27,7 +27,7 @@ from freqtrade.misc import get_strategy_run_id from freqtrade.mixins import LoggingMixin from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, - store_backtest_stats) + store_backtest_stats, store_backtest_signal_candles) from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager @@ -1157,6 +1157,9 @@ class Backtesting: if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], self.results) + if self.enable_backtest_signal_candle_export and self.dataprovider.runmode == RunMode.BACKTEST: + store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs) + # Results may be mixed up now. Sort them so they follow --strategy-list order. if 'strategy_list' in self.config and len(self.results) > 0: self.results['strategy_comparison'] = sorted( From b1bcf9f33c8157fc1e62fa67532151afaaacb8f2 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 16 Apr 2022 14:58:17 +0100 Subject: [PATCH 05/23] Fix backtest_enable typo --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4ed3f85be..f19cd488e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1157,7 +1157,7 @@ class Backtesting: if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], self.results) - if self.enable_backtest_signal_candle_export and self.dataprovider.runmode == RunMode.BACKTEST: + if self.backtest_signal_candle_export_enable and self.dataprovider.runmode == RunMode.BACKTEST: store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs) # Results may be mixed up now. Sort them so they follow --strategy-list order. From f55a9940a7e812f07437e4a207652b8152d7d131 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 16 Apr 2022 16:15:04 +0100 Subject: [PATCH 06/23] Fix line spacing --- freqtrade/optimize/optimize_reports.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 4265ca70d..f870bd1f5 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -45,6 +45,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN) file_dump_json(latest_filename, {'latest_backtest': str(filename.name)}) + def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> None: """ Stores backtest trade signal candles @@ -65,6 +66,7 @@ def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict] with open(filename, 'wb') as f: pickle.dump(candles, f) + def _get_line_floatfmt(stake_currency: str) -> List[str]: """ Generate floatformat (goes in line with _generate_result_line()) From a63affc5f14156b671c805de9256d3e611d05c93 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 16 Apr 2022 16:32:04 +0100 Subject: [PATCH 07/23] Fix flake8 complaints --- freqtrade/optimize/backtesting.py | 13 ++++++++----- freqtrade/optimize/optimize_reports.py | 3 ++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f19cd488e..7e19e26e4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -8,7 +8,6 @@ from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple -import pickle from numpy import nan from pandas import DataFrame @@ -20,14 +19,16 @@ from freqtrade.data import history from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider -from freqtrade.enums import BacktestState, CandleType, ExitCheckTuple, ExitType, TradingMode, RunMode +from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType, TradingMode, + RunMode) from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import get_strategy_run_id from freqtrade.mixins import LoggingMixin from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, - store_backtest_stats, store_backtest_signal_candles) + store_backtest_stats, + store_backtest_signal_candles) from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager @@ -1075,7 +1076,8 @@ class Backtesting: }) self.all_results[self.strategy.get_strategy_name()] = results - if self.backtest_signal_candle_export_enable and self.dataprovider.runmode == RunMode.BACKTEST: + if self.backtest_signal_candle_export_enable and + self.dataprovider.runmode == RunMode.BACKTEST: signal_candles_only = {} for pair in preprocessed_tmp.keys(): signal_candles_only_df = DataFrame() @@ -1157,7 +1159,8 @@ class Backtesting: if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], self.results) - if self.backtest_signal_candle_export_enable and self.dataprovider.runmode == RunMode.BACKTEST: + if self.backtest_signal_candle_export_enable and + self.dataprovider.runmode == RunMode.BACKTEST: store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs) # Results may be mixed up now. Sort them so they follow --strategy-list order. diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index f870bd1f5..06b393b60 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -51,7 +51,8 @@ def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict] Stores backtest trade signal candles :param recordfilename: Path object, which can either be a filename or a directory. Filenames will be appended with a timestamp right before the suffix - while for directories, /backtest-result-_signals.pkl will be used as filename + while for directories, /backtest-result-_signals.pkl will be used + as filename :param stats: Dict containing the backtesting signal candles """ if recordfilename.is_dir(): From 7210a1173074915b76751085ff3df2f23b2a55aa Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 16 Apr 2022 16:37:06 +0100 Subject: [PATCH 08/23] Fix flake8 complaints --- freqtrade/optimize/backtesting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 7e19e26e4..839463218 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -131,7 +131,8 @@ class Backtesting: self.config['startup_candle_count'] = self.required_startup self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) - self.backtest_signal_candle_export_enable = self.config.get('backtest_signal_candle_export_enable', False) + self.backtest_signal_candle_export_enable = self.config.get( + 'backtest_signal_candle_export_enable', False) self.trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) # strategies which define "can_short=True" will fail to load in Spot mode. From b738c4e695e69d51a3827f3f99b42e40f0b01846 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 16 Apr 2022 16:49:20 +0100 Subject: [PATCH 09/23] Fix flake8 complaints --- freqtrade/optimize/backtesting.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 839463218..e9440f62c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -20,7 +20,7 @@ from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_t from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType, TradingMode, - RunMode) + RunMode) from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import get_strategy_run_id @@ -1077,8 +1077,8 @@ class Backtesting: }) self.all_results[self.strategy.get_strategy_name()] = results - if self.backtest_signal_candle_export_enable and - self.dataprovider.runmode == RunMode.BACKTEST: + if self.backtest_signal_candle_export_enable and \ + self.dataprovider.runmode == RunMode.BACKTEST: signal_candles_only = {} for pair in preprocessed_tmp.keys(): signal_candles_only_df = DataFrame() @@ -1160,8 +1160,8 @@ class Backtesting: if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], self.results) - if self.backtest_signal_candle_export_enable and - self.dataprovider.runmode == RunMode.BACKTEST: + if self.backtest_signal_candle_export_enable and \ + self.dataprovider.runmode == RunMode.BACKTEST: store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs) # Results may be mixed up now. Sort them so they follow --strategy-list order. From 34fb8dacd7fb9083461ad2986b47d2d8480a7e05 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 16 Apr 2022 17:03:24 +0100 Subject: [PATCH 10/23] Fix isort complaints --- freqtrade/optimize/backtesting.py | 8 ++++---- freqtrade/optimize/optimize_reports.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e9440f62c..03a6cade0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -19,16 +19,16 @@ from freqtrade.data import history from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider -from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType, TradingMode, - RunMode) +from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType, RunMode, + TradingMode) from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import get_strategy_run_id from freqtrade.mixins import LoggingMixin from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, - store_backtest_stats, - store_backtest_signal_candles) + store_backtest_signal_candles, + store_backtest_stats) from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 06b393b60..f0b2e2e71 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -1,9 +1,9 @@ import logging +import pickle from copy import deepcopy from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Dict, List, Union -import pickle from numpy import int64 from pandas import DataFrame, to_datetime From 84f486295d806d4ae8347582f7d3d803bf41d256 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 19 Apr 2022 12:48:21 +0100 Subject: [PATCH 11/23] Add tests for new storing of backtest signal candles --- freqtrade/misc.py | 16 ++++++++++++++++ freqtrade/optimize/optimize_reports.py | 10 ++++------ tests/optimize/test_optimize_reports.py | 24 +++++++++++++++++++++++- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index d5572ea0b..9087ec6e2 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -4,6 +4,7 @@ Various tool function for Freqtrade and scripts import gzip import hashlib import logging +import pickle import re from copy import deepcopy from datetime import datetime @@ -86,6 +87,21 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = logger.debug(f'done json to "{filename}"') +def file_dump_pickle(filename: Path, data: Any, log: bool = True) -> None: + """ + Dump object data into a file + :param filename: file to create + :param data: Object data to save + :return: + """ + + if log: + logger.info(f'dumping pickle to "{filename}"') + with open(filename, 'wb') as fp: + pickle.dump(data, fp) + logger.debug(f'done pickling to "{filename}"') + + def json_load(datafile: IO) -> Any: """ load data with rapidjson diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index f0b2e2e71..ed29af839 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -1,5 +1,4 @@ import logging -import pickle from copy import deepcopy from datetime import datetime, timedelta, timezone from pathlib import Path @@ -12,8 +11,8 @@ from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change, calculate_max_drawdown) -from freqtrade.misc import (decimals_per_coin, file_dump_json, get_backtest_metadata_filename, - round_coin_value) +from freqtrade.misc import (decimals_per_coin, file_dump_json, file_dump_pickle, + get_backtest_metadata_filename, round_coin_value) logger = logging.getLogger(__name__) @@ -61,11 +60,10 @@ def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict] else: filename = Path.joinpath( recordfilename.parent, - f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' + f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals' ).with_suffix(recordfilename.suffix) - with open(filename, 'wb') as f: - pickle.dump(candles, f) + file_dump_pickle(filename, candles) def _get_line_floatfmt(stake_currency: str) -> List[str]: diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 05c0bf575..a09620a71 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -1,5 +1,5 @@ import re -from datetime import timedelta +from datetime import timedelta, timezone from pathlib import Path import pandas as pd @@ -19,6 +19,7 @@ from freqtrade.optimize.optimize_reports import (_get_resample_from_period, gene generate_periodic_breakdown_stats, generate_strategy_comparison, generate_trading_stats, show_sorted_pairlist, + store_backtest_signal_candles, store_backtest_stats, text_table_bt_results, text_table_exit_reason, text_table_strategy) from freqtrade.resolvers.strategy_resolver import StrategyResolver @@ -201,6 +202,27 @@ def test_store_backtest_stats(testdatadir, mocker): assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / 'testresult')) +def test_store_backtest_candles(testdatadir, mocker): + + dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_pickle') + + # test directory exporting + store_backtest_signal_candles(testdatadir, {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}}) + + assert dump_mock.call_count == 1 + assert isinstance(dump_mock.call_args_list[0][0][0], Path) + assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl')) + + dump_mock.reset_mock() + # test file exporting + filename = testdatadir / 'testresult' + store_backtest_signal_candles(filename, {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}}) + assert dump_mock.call_count == 1 + assert isinstance(dump_mock.call_args_list[0][0][0], Path) + # result will be testdatadir / testresult-_signals.pkl + assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl')) + + def test_generate_pair_metrics(): results = pd.DataFrame( From aa5984930d3cb19b7fc2ed82b5590a6bad640d42 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 19 Apr 2022 13:00:09 +0100 Subject: [PATCH 12/23] Fix filename generation --- freqtrade/optimize/optimize_reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index ed29af839..05eec693e 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -60,8 +60,8 @@ def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict] else: filename = Path.joinpath( recordfilename.parent, - f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals' - ).with_suffix(recordfilename.suffix) + f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl' + ) file_dump_pickle(filename, candles) From 3ad1411f5ecba585be2a23972d412f692da15018 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 19 Apr 2022 13:08:01 +0100 Subject: [PATCH 13/23] Fix imports --- tests/optimize/test_optimize_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index a09620a71..d72cf4e86 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -1,5 +1,5 @@ import re -from datetime import timedelta, timezone +from datetime import timedelta from pathlib import Path import pandas as pd From 9421d19cba64813a16094ff3bf2d332641e3fd89 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 19 Apr 2022 14:05:03 +0100 Subject: [PATCH 14/23] Add documentation --- docs/configuration.md | 7 +-- docs/strategy_analysis_example.md | 74 ++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 369c4e2dd..061b6c77c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,7 +11,7 @@ Per default, the bot loads the configuration from the `config.json` file, locate You can specify a different configuration file used by the bot with the `-c/--config` command-line option. -If you used the [Quick start](installation.md/#quick-start) method for installing +If you used the [Quick start](installation.md/#quick-start) method for installing the bot, the installation script should have already created the default configuration file (`config.json`) for you. If the default configuration file is not created we recommend to use `freqtrade new-config --config config.json` to generate a basic configuration file. @@ -64,7 +64,7 @@ This is similar to using multiple `--config` parameters, but simpler in usage as "config-private.json" ] ``` - + ``` bash freqtrade trade --config user_data/config.json <...> ``` @@ -100,7 +100,7 @@ This is similar to using multiple `--config` parameters, but simpler in usage as "stake_amount": "unlimited", } ``` - + Resulting combined configuration: ``` json title="Result" @@ -229,6 +229,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String | `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `-1`.*
**Datatype:** Positive Integer or -1 +| `backtest_signal_candle_export_enable` | Enables the exporting of signal candles for use in post-backtesting analysis of buy tags. See [Strategy Analysis](strategy_analysis_example.md#analyse-the-buy-entry-and-sell-exit-tags).
*Defaults to `false`.*
**Datatype:** Boolean ### Parameters in the strategy diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 2fa84a6df..48f54c824 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -93,7 +93,7 @@ from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats # if backtest_dir points to a directory, it'll automatically load the last backtest file. backtest_dir = config["user_data_dir"] / "backtest_results" -# backtest_dir can also point to a specific file +# backtest_dir can also point to a specific file # backtest_dir = config["user_data_dir"] / "backtest_results/backtest-result-2020-07-01_20-04-22.json" ``` @@ -250,3 +250,75 @@ fig.show() ``` Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data. + +## Analyse the buy/entry and sell/exit tags + +It can be helpful to understand how a strategy behaves according to the buy/entry tags used to +mark up different buy conditions. You might want to see more complex statistics about each buy and +sell condition above those provided by the default backtesting output. You may also want to +determine indicator values on the signal candle that resulted in a trade opening. + +We first need to enable the exporting of trades from backtesting: + +``` +freqtrade backtesting -c --timeframe --strategy --timerange= --export=trades --export-filename=user_data/backtest_results/- +``` + +To analyse the buy tags, we need to use the buy_reasons.py script in the `scripts/` +folder. We need the signal candles for each opened trade so add the following option to your +config file: + +``` +'backtest_signal_candle_export_enable': true, +``` + +This will tell freqtrade to output a pickled dictionary of strategy, pairs and corresponding +DataFrame of the candles that resulted in buy signals. Depending on how many buys your strategy +makes, this file may get quite large, so periodically check your `user_data/backtest_results` +folder to delete old exports. + +Before running your next backtest, make sure you either delete your old backtest results or run +backtesting with the `--cache none` option to make sure no cached results are used. + +If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the +`user_data/backtest_results` folder. + +Now run the buy_reasons.py script, supplying a few options: + +``` +./scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 +``` + +The `-g` option is used to specify the various tabular outputs, ranging from the simplest (0) +to the most detailed per pair, per buy and per sell tag (4). More options are available by +running with the `-h` option. + +### Tuning the buy tags and sell tags to display + +To show only certain buy and sell tags in the displayed output, use the following two options: + +``` +--buy_reason_list : Comma separated list of buy signals to analyse. Default: "all" +--sell_reason_list : Comma separated list of sell signals to analyse. Default: "stop_loss,trailing_stop_loss" +``` + +For example: + +``` +./scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" +``` + +### Outputting signal candle indicators + +The real power of the buy_reasons.py script comes from the ability to print out the indicator +values present on signal candles to allow fine-grained investigation and tuning of buy signal +indicators. To print out a column for a given set of indicators, use the `--indicator-list` +option: + +``` +./scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +``` + +The indicators have to be present in your strategy's main dataframe (either for your main +timeframe or for informatives) otherwise they will simply be ignored in the script +output. From b3cb7226467c359b0748cb4866f8f8fcfa649a53 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 20 Apr 2022 13:38:52 +0100 Subject: [PATCH 15/23] Use joblib instead of pickle, add signal candle read/write test, move docs to new Advanced Backtesting doc --- docs/advanced-backtesting.md | 75 +++++++++++++++++++++++++ docs/data-analysis.md | 1 + docs/strategy_analysis_example.md | 72 ------------------------ freqtrade/misc.py | 10 ++-- freqtrade/optimize/optimize_reports.py | 8 ++- tests/optimize/test_optimize_reports.py | 48 ++++++++++++++-- 6 files changed, 128 insertions(+), 86 deletions(-) create mode 100644 docs/advanced-backtesting.md diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md new file mode 100644 index 000000000..9a7d71767 --- /dev/null +++ b/docs/advanced-backtesting.md @@ -0,0 +1,75 @@ +# Advanced Backtesting Analysis + +## Analyse the buy/entry and sell/exit tags + +It can be helpful to understand how a strategy behaves according to the buy/entry tags used to +mark up different buy conditions. You might want to see more complex statistics about each buy and +sell condition above those provided by the default backtesting output. You may also want to +determine indicator values on the signal candle that resulted in a trade opening. + +!!! Note + The following buy reason analysis is only available for backtesting, *not hyperopt*. + +We first need to enable the exporting of trades from backtesting: + +```bash +freqtrade backtesting -c --timeframe --strategy --timerange= --export=trades --export-filename=user_data/backtest_results/- +``` + +To analyse the buy tags, we need to use the `freqtrade tag-analysis` command. We need the signal +candles for each opened trade so add the following option to your config file: + +``` +'backtest_signal_candle_export_enable': true, +``` + +This will tell freqtrade to output a pickled dictionary of strategy, pairs and corresponding +DataFrame of the candles that resulted in buy signals. Depending on how many buys your strategy +makes, this file may get quite large, so periodically check your `user_data/backtest_results` +folder to delete old exports. + +Before running your next backtest, make sure you either delete your old backtest results or run +backtesting with the `--cache none` option to make sure no cached results are used. + +If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the +`user_data/backtest_results` folder. + +Now run the buy_reasons.py script, supplying a few options: + +```bash +freqtrade tag-analysis -c -s -t -g0,1,2,3,4 +``` + +The `-g` option is used to specify the various tabular outputs, ranging from the simplest (0) +to the most detailed per pair, per buy and per sell tag (4). More options are available by +running with the `-h` option. + +### Tuning the buy tags and sell tags to display + +To show only certain buy and sell tags in the displayed output, use the following two options: + +``` +--buy_reason_list : Comma separated list of buy signals to analyse. Default: "all" +--sell_reason_list : Comma separated list of sell signals to analyse. Default: "stop_loss,trailing_stop_loss" +``` + +For example: + +```bash +freqtrade tag-analysis -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" +``` + +### Outputting signal candle indicators + +The real power of the buy_reasons.py script comes from the ability to print out the indicator +values present on signal candles to allow fine-grained investigation and tuning of buy signal +indicators. To print out a column for a given set of indicators, use the `--indicator-list` +option: + +```bash +freqtrade tag-analysis -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +``` + +The indicators have to be present in your strategy's main DataFrame (either for your main +timeframe or for informatives) otherwise they will simply be ignored in the script +output. diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 9a79ee5ed..926ed3eae 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -122,5 +122,6 @@ Best avoid relative paths, since this starts at the storage location of the jupy * [Strategy debugging](strategy_analysis_example.md) - also available as Jupyter notebook (`user_data/notebooks/strategy_analysis_example.ipynb`) * [Plotting](plotting.md) +* [Tag Analysis](advanced-backtesting.md) Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data. diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 48f54c824..ae0c6a6a3 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -250,75 +250,3 @@ fig.show() ``` Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data. - -## Analyse the buy/entry and sell/exit tags - -It can be helpful to understand how a strategy behaves according to the buy/entry tags used to -mark up different buy conditions. You might want to see more complex statistics about each buy and -sell condition above those provided by the default backtesting output. You may also want to -determine indicator values on the signal candle that resulted in a trade opening. - -We first need to enable the exporting of trades from backtesting: - -``` -freqtrade backtesting -c --timeframe --strategy --timerange= --export=trades --export-filename=user_data/backtest_results/- -``` - -To analyse the buy tags, we need to use the buy_reasons.py script in the `scripts/` -folder. We need the signal candles for each opened trade so add the following option to your -config file: - -``` -'backtest_signal_candle_export_enable': true, -``` - -This will tell freqtrade to output a pickled dictionary of strategy, pairs and corresponding -DataFrame of the candles that resulted in buy signals. Depending on how many buys your strategy -makes, this file may get quite large, so periodically check your `user_data/backtest_results` -folder to delete old exports. - -Before running your next backtest, make sure you either delete your old backtest results or run -backtesting with the `--cache none` option to make sure no cached results are used. - -If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the -`user_data/backtest_results` folder. - -Now run the buy_reasons.py script, supplying a few options: - -``` -./scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 -``` - -The `-g` option is used to specify the various tabular outputs, ranging from the simplest (0) -to the most detailed per pair, per buy and per sell tag (4). More options are available by -running with the `-h` option. - -### Tuning the buy tags and sell tags to display - -To show only certain buy and sell tags in the displayed output, use the following two options: - -``` ---buy_reason_list : Comma separated list of buy signals to analyse. Default: "all" ---sell_reason_list : Comma separated list of sell signals to analyse. Default: "stop_loss,trailing_stop_loss" -``` - -For example: - -``` -./scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" -``` - -### Outputting signal candle indicators - -The real power of the buy_reasons.py script comes from the ability to print out the indicator -values present on signal candles to allow fine-grained investigation and tuning of buy signal -indicators. To print out a column for a given set of indicators, use the `--indicator-list` -option: - -``` -./scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" -``` - -The indicators have to be present in your strategy's main dataframe (either for your main -timeframe or for informatives) otherwise they will simply be ignored in the script -output. diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 9087ec6e2..be12d8224 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -4,7 +4,6 @@ Various tool function for Freqtrade and scripts import gzip import hashlib import logging -import pickle import re from copy import deepcopy from datetime import datetime @@ -13,6 +12,7 @@ from typing import Any, Iterator, List, Union from typing.io import IO from urllib.parse import urlparse +import joblib import rapidjson from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN @@ -87,7 +87,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = logger.debug(f'done json to "{filename}"') -def file_dump_pickle(filename: Path, data: Any, log: bool = True) -> None: +def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None: """ Dump object data into a file :param filename: file to create @@ -96,10 +96,10 @@ def file_dump_pickle(filename: Path, data: Any, log: bool = True) -> None: """ if log: - logger.info(f'dumping pickle to "{filename}"') + logger.info(f'dumping joblib to "{filename}"') with open(filename, 'wb') as fp: - pickle.dump(data, fp) - logger.debug(f'done pickling to "{filename}"') + joblib.dump(data, fp) + logger.debug(f'done joblib dump to "{filename}"') def json_load(datafile: IO) -> Any: diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 05eec693e..6288ee16a 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -11,7 +11,7 @@ from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change, calculate_max_drawdown) -from freqtrade.misc import (decimals_per_coin, file_dump_json, file_dump_pickle, +from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json, get_backtest_metadata_filename, round_coin_value) @@ -45,7 +45,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N file_dump_json(latest_filename, {'latest_backtest': str(filename.name)}) -def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> None: +def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> Path: """ Stores backtest trade signal candles :param recordfilename: Path object, which can either be a filename or a directory. @@ -63,7 +63,9 @@ def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict] f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl' ) - file_dump_pickle(filename, candles) + file_dump_joblib(filename, candles) + + return filename def _get_line_floatfmt(stake_currency: str) -> List[str]: diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index d72cf4e86..ff8d420b3 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -2,6 +2,7 @@ import re from datetime import timedelta from pathlib import Path +import joblib import pandas as pd import pytest from arrow import Arrow @@ -204,23 +205,58 @@ def test_store_backtest_stats(testdatadir, mocker): def test_store_backtest_candles(testdatadir, mocker): - dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_pickle') + dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_joblib') - # test directory exporting - store_backtest_signal_candles(testdatadir, {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}}) + candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} + + # mock directory exporting + store_backtest_signal_candles(testdatadir, candle_dict) assert dump_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl')) dump_mock.reset_mock() - # test file exporting - filename = testdatadir / 'testresult' - store_backtest_signal_candles(filename, {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}}) + # mock file exporting + filename = Path(testdatadir / 'testresult') + store_backtest_signal_candles(filename, candle_dict) assert dump_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) # result will be testdatadir / testresult-_signals.pkl assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl')) + dump_mock.reset_mock() + + +def test_write_read_backtest_candles(tmpdir): + + candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} + + # test directory exporting + stored_file = store_backtest_signal_candles(Path(tmpdir), candle_dict) + scp = open(stored_file, "rb") + pickled_signal_candles = joblib.load(scp) + scp.close() + + assert pickled_signal_candles.keys() == candle_dict.keys() + assert pickled_signal_candles['DefStrat'].keys() == pickled_signal_candles['DefStrat'].keys() + assert pickled_signal_candles['DefStrat']['UNITTEST/BTC'] \ + .equals(pickled_signal_candles['DefStrat']['UNITTEST/BTC']) + + _clean_test_file(stored_file) + + # test file exporting + filename = Path(tmpdir / 'testresult') + stored_file = store_backtest_signal_candles(filename, candle_dict) + scp = open(stored_file, "rb") + pickled_signal_candles = joblib.load(scp) + scp.close() + + assert pickled_signal_candles.keys() == candle_dict.keys() + assert pickled_signal_candles['DefStrat'].keys() == pickled_signal_candles['DefStrat'].keys() + assert pickled_signal_candles['DefStrat']['UNITTEST/BTC'] \ + .equals(pickled_signal_candles['DefStrat']['UNITTEST/BTC']) + + _clean_test_file(stored_file) def test_generate_pair_metrics(): From ea7fb4e6e6182c71aabcaf0fea9bf52214644b90 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 20 Apr 2022 13:51:45 +0100 Subject: [PATCH 16/23] Revert docs to buy_reasons script version --- docs/advanced-backtesting.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index 9a7d71767..d8c6c505e 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -13,11 +13,15 @@ determine indicator values on the signal candle that resulted in a trade opening We first need to enable the exporting of trades from backtesting: ```bash -freqtrade backtesting -c --timeframe --strategy --timerange= --export=trades --export-filename=user_data/backtest_results/- +freqtrade backtesting -c --timeframe --strategy --timerange= --export=trades ``` -To analyse the buy tags, we need to use the `freqtrade tag-analysis` command. We need the signal -candles for each opened trade so add the following option to your config file: +To analyse the buy tags, we need to use the `buy_reasons.py` script from +[froggleston's repo](https://github.com/froggleston/freqtrade-buyreasons). Follow the instructions +in their README to copy the script into your `freqtrade/scripts/` folder. + +We then need the signal candles for each opened trade so add the following option to your +config file: ``` 'backtest_signal_candle_export_enable': true, @@ -37,7 +41,7 @@ If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` Now run the buy_reasons.py script, supplying a few options: ```bash -freqtrade tag-analysis -c -s -t -g0,1,2,3,4 +python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 ``` The `-g` option is used to specify the various tabular outputs, ranging from the simplest (0) @@ -56,7 +60,7 @@ To show only certain buy and sell tags in the displayed output, use the followin For example: ```bash -freqtrade tag-analysis -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" +python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" ``` ### Outputting signal candle indicators @@ -67,7 +71,7 @@ indicators. To print out a column for a given set of indicators, use the `--indi option: ```bash -freqtrade tag-analysis -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --buy_reason_list "buy_tag_a,buy_tag_b" --sell_reason_list "roi,custom_sell_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" ``` The indicators have to be present in your strategy's main DataFrame (either for your main From 933054a51ccb991a646452e2e26f2ea7cad37229 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 20 Apr 2022 13:54:50 +0100 Subject: [PATCH 17/23] Move enable option text to make better sense --- docs/advanced-backtesting.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index d8c6c505e..4d91c4305 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -10,7 +10,15 @@ determine indicator values on the signal candle that resulted in a trade opening !!! Note The following buy reason analysis is only available for backtesting, *not hyperopt*. -We first need to enable the exporting of trades from backtesting: +We first need to tell freqtrade to export the signal candles for each opened trade, +so add the following option to your config file: + +``` +'backtest_signal_candle_export_enable': true, +``` + +We then need to run backtesting and include the `--export` option to enable the exporting of +trades: ```bash freqtrade backtesting -c --timeframe --strategy --timerange= --export=trades @@ -20,13 +28,6 @@ To analyse the buy tags, we need to use the `buy_reasons.py` script from [froggleston's repo](https://github.com/froggleston/freqtrade-buyreasons). Follow the instructions in their README to copy the script into your `freqtrade/scripts/` folder. -We then need the signal candles for each opened trade so add the following option to your -config file: - -``` -'backtest_signal_candle_export_enable': true, -``` - This will tell freqtrade to output a pickled dictionary of strategy, pairs and corresponding DataFrame of the candles that resulted in buy signals. Depending on how many buys your strategy makes, this file may get quite large, so periodically check your `user_data/backtest_results` From f92997d3789c6eb8b9116e4a55dac2b9b7112aa7 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 20 Apr 2022 14:05:33 +0100 Subject: [PATCH 18/23] Move signal candle generation into separate function --- freqtrade/optimize/backtesting.py | 37 +++++++++++++++++-------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 03a6cade0..9d7f19f7a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1079,26 +1079,29 @@ class Backtesting: if self.backtest_signal_candle_export_enable and \ self.dataprovider.runmode == RunMode.BACKTEST: - signal_candles_only = {} - for pair in preprocessed_tmp.keys(): - signal_candles_only_df = DataFrame() - - pairdf = preprocessed_tmp[pair] - resdf = results['results'] - pairresults = resdf.loc[(resdf["pair"] == pair)] - - if pairdf.shape[0] > 0: - for t, v in pairresults.open_date.items(): - allinds = pairdf.loc[(pairdf['date'] < v)] - signal_inds = allinds.iloc[[-1]] - signal_candles_only_df = signal_candles_only_df.append(signal_inds) - - signal_candles_only[pair] = signal_candles_only_df - - self.processed_dfs[self.strategy.get_strategy_name()] = signal_candles_only + self._generate_trade_signal_candles(preprocessed_tmp, results) return min_date, max_date + def _generate_trade_signal_candles(self, preprocessed_df, bt_results): + signal_candles_only = {} + for pair in preprocessed_df.keys(): + signal_candles_only_df = DataFrame() + + pairdf = preprocessed_df[pair] + resdf = bt_results['results'] + pairresults = resdf.loc[(resdf["pair"] == pair)] + + if pairdf.shape[0] > 0: + for t, v in pairresults.open_date.items(): + allinds = pairdf.loc[(pairdf['date'] < v)] + signal_inds = allinds.iloc[[-1]] + signal_candles_only_df = signal_candles_only_df.append(signal_inds) + + signal_candles_only[pair] = signal_candles_only_df + + self.processed_dfs[self.strategy.get_strategy_name()] = signal_candles_only + def _get_min_cached_backtest_date(self): min_backtest_date = None backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT) From 7f60364f63057bba06dfc3bf3ed3886560906a3b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 22 Apr 2022 06:38:51 +0200 Subject: [PATCH 19/23] Add doc-page to index --- docs/advanced-backtesting.md | 12 ++++++------ freqtrade/optimize/backtesting.py | 8 ++++---- mkdocs.yml | 1 + 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index 4d91c4305..69dc428f1 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -1,6 +1,6 @@ # Advanced Backtesting Analysis -## Analyse the buy/entry and sell/exit tags +## Analyze the buy/entry and sell/exit tags It can be helpful to understand how a strategy behaves according to the buy/entry tags used to mark up different buy conditions. You might want to see more complex statistics about each buy and @@ -20,11 +20,11 @@ so add the following option to your config file: We then need to run backtesting and include the `--export` option to enable the exporting of trades: -```bash +``` bash freqtrade backtesting -c --timeframe --strategy --timerange= --export=trades ``` -To analyse the buy tags, we need to use the `buy_reasons.py` script from +To analyze the buy tags, we need to use the `buy_reasons.py` script from [froggleston's repo](https://github.com/froggleston/freqtrade-buyreasons). Follow the instructions in their README to copy the script into your `freqtrade/scripts/` folder. @@ -39,9 +39,9 @@ backtesting with the `--cache none` option to make sure no cached results are us If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the `user_data/backtest_results` folder. -Now run the buy_reasons.py script, supplying a few options: +Now run the `buy_reasons.py` script, supplying a few options: -```bash +``` bash python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 ``` @@ -76,5 +76,5 @@ python3 scripts/buy_reasons.py -c -s -t Date: Fri, 22 Apr 2022 18:46:12 +0100 Subject: [PATCH 20/23] Add signals enum to 'export' cli option --- freqtrade/commands/arguments.py | 2 +- freqtrade/constants.py | 3 +-- freqtrade/optimize/backtesting.py | 5 +++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 7d4624bd1..8a108fe79 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -23,7 +23,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", "enable_protections", "dry_run_wallet", "timeframe_detail", - "strategy_list", "export", "exportfilename", + "strategy_list", "export", "exportfilename" "backtest_breakdown", "backtest_cache"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d21020a3f..1a21ec77f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -14,7 +14,7 @@ PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec TIMEOUT_UNITS = ['minutes', 'seconds'] -EXPORT_OPTIONS = ['none', 'trades'] +EXPORT_OPTIONS = ['none', 'trades', 'signals'] DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' @@ -380,7 +380,6 @@ CONF_SCHEMA = { }, 'position_adjustment_enable': {'type': 'boolean'}, 'max_entry_position_adjustment': {'type': ['integer', 'number'], 'minimum': -1}, - 'backtest_signal_candle_export_enable': {'type': 'boolean'}, }, 'definitions': { 'exchange': { diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9d7f19f7a..c2d9f1edb 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1077,7 +1077,7 @@ class Backtesting: }) self.all_results[self.strategy.get_strategy_name()] = results - if self.backtest_signal_candle_export_enable and \ + if self.config.get('export', 'none') == 'signals' and \ self.dataprovider.runmode == RunMode.BACKTEST: self._generate_trade_signal_candles(preprocessed_tmp, results) @@ -1163,8 +1163,9 @@ class Backtesting: if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], self.results) - if self.backtest_signal_candle_export_enable and \ + if self.config.get('export', 'none') == 'signals' and \ self.dataprovider.runmode == RunMode.BACKTEST: + store_backtest_stats(self.config['exportfilename'], self.results) store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs) # Results may be mixed up now. Sort them so they follow --strategy-list order. From 2fc4e5e1172917368719fc82ec16e537298d55dc Mon Sep 17 00:00:00 2001 From: froggleston Date: Fri, 22 Apr 2022 18:54:02 +0100 Subject: [PATCH 21/23] Fix weird removal of comma --- freqtrade/commands/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 8a108fe79..7d4624bd1 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -23,7 +23,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", "enable_protections", "dry_run_wallet", "timeframe_detail", - "strategy_list", "export", "exportfilename" + "strategy_list", "export", "exportfilename", "backtest_breakdown", "backtest_cache"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", From dff9d52b3067a843e48c57e56e70a3942611f88d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Apr 2022 08:51:52 +0200 Subject: [PATCH 22/23] Remove hints on no longer used option, add very primitive test --- docs/backtesting.md | 24 ++++++++++++------------ docs/configuration.md | 1 - freqtrade/optimize/backtesting.py | 11 ++++------- tests/optimize/test_backtesting.py | 5 ++++- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 5d836d01b..f732068f1 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -20,7 +20,8 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--dry-run-wallet DRY_RUN_WALLET] [--timeframe-detail TIMEFRAME_DETAIL] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] - [--export {none,trades}] [--export-filename PATH] + [--export {none,trades,signals}] + [--export-filename PATH] [--breakdown {day,week,month} [{day,week,month} ...]] [--cache {none,day,week,month}] @@ -63,18 +64,17 @@ optional arguments: `30m`, `1h`, `1d`). --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] Provide a space-separated list of strategies to - backtest. Please note that timeframe needs to be - set either in config or via command line. When using - this together with `--export trades`, the strategy- - name is injected into the filename (so `backtest- - data.json` becomes `backtest-data-SampleStrategy.json` - --export {none,trades} + backtest. Please note that timeframe needs to be set + either in config or via command line. When using this + together with `--export trades`, the strategy-name is + injected into the filename (so `backtest-data.json` + becomes `backtest-data-SampleStrategy.json` + --export {none,trades,signals} Export backtest results (default: trades). - --export-filename PATH - Save backtest results to the file with this filename. - Requires `--export` to be set as well. Example: - `--export-filename=user_data/backtest_results/backtest - _today.json` + --export-filename PATH, --backtest-filename PATH + Use this filename for backtest results.Requires + `--export` to be set as well. Example: `--export-filen + ame=user_data/backtest_results/backtest_today.json` --breakdown {day,week,month} [{day,week,month} ...] Show backtesting breakdown per [day, week, month]. --cache {none,day,week,month} diff --git a/docs/configuration.md b/docs/configuration.md index 061b6c77c..5770450a6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -229,7 +229,6 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String | `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `-1`.*
**Datatype:** Positive Integer or -1 -| `backtest_signal_candle_export_enable` | Enables the exporting of signal candles for use in post-backtesting analysis of buy tags. See [Strategy Analysis](strategy_analysis_example.md#analyse-the-buy-entry-and-sell-exit-tags).
*Defaults to `false`.*
**Datatype:** Boolean ### Parameters in the strategy diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c2d9f1edb..f4149fdc1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -131,9 +131,6 @@ class Backtesting: self.config['startup_candle_count'] = self.required_startup self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) - self.backtest_signal_candle_export_enable = self.config.get( - 'backtest_signal_candle_export_enable', False) - self.trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) # strategies which define "can_short=True" will fail to load in Spot mode. self._can_short = self.trading_mode != TradingMode.SPOT @@ -1077,8 +1074,8 @@ class Backtesting: }) self.all_results[self.strategy.get_strategy_name()] = results - if self.config.get('export', 'none') == 'signals' and \ - self.dataprovider.runmode == RunMode.BACKTEST: + if (self.config.get('export', 'none') == 'signals' and + self.dataprovider.runmode == RunMode.BACKTEST): self._generate_trade_signal_candles(preprocessed_tmp, results) return min_date, max_date @@ -1163,8 +1160,8 @@ class Backtesting: if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], self.results) - if self.config.get('export', 'none') == 'signals' and \ - self.dataprovider.runmode == RunMode.BACKTEST: + if (self.config.get('export', 'none') == 'signals' and + self.dataprovider.runmode == RunMode.BACKTEST): store_backtest_stats(self.config['exportfilename'], self.results) store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 08957acf9..797d3bafa 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -384,14 +384,16 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats') mocker.patch('freqtrade.optimize.backtesting.show_backtest_results') sbs = mocker.patch('freqtrade.optimize.backtesting.store_backtest_stats') + sbc = mocker.patch('freqtrade.optimize.backtesting.store_backtest_signal_candles') mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) default_conf['timeframe'] = '1m' default_conf['datadir'] = testdatadir - default_conf['export'] = 'trades' + default_conf['export'] = 'signals' default_conf['exportfilename'] = 'export.txt' default_conf['timerange'] = '-1510694220' + default_conf['runmode'] = RunMode.BACKTEST backtesting = Backtesting(default_conf) backtesting._set_strategy(backtesting.strategylist[0]) @@ -407,6 +409,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: assert backtesting.strategy.dp._pairlists is not None assert backtesting.strategy.bot_loop_start.call_count == 1 assert sbs.call_count == 1 + assert sbc.call_count == 1 def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: From 580a2c6545ce79ffda7b54f2e1a928426d519aef Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Apr 2022 09:23:53 +0200 Subject: [PATCH 23/23] Don't repeat backtest-storing --- freqtrade/optimize/backtesting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f4149fdc1..f5571c4e2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1157,12 +1157,11 @@ class Backtesting: else: self.results = results - if self.config.get('export', 'none') == 'trades': + if self.config.get('export', 'none') in ('trades', 'signals'): store_backtest_stats(self.config['exportfilename'], self.results) if (self.config.get('export', 'none') == 'signals' and self.dataprovider.runmode == RunMode.BACKTEST): - store_backtest_stats(self.config['exportfilename'], self.results) store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs) # Results may be mixed up now. Sort them so they follow --strategy-list order.