From b77f926cdd91d41779534f5312b196851f58212e Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 4 Sep 2023 10:53:04 +0900 Subject: [PATCH 01/52] add recursive analysis --- freqtrade/commands/__init__.py | 3 +- freqtrade/commands/arguments.py | 15 +- freqtrade/commands/optimize_commands.py | 12 + freqtrade/optimize/recursive_analysis.py | 236 ++++++++++++++++++ .../optimize/recursive_analysis_helpers.py | 182 ++++++++++++++ 5 files changed, 446 insertions(+), 2 deletions(-) create mode 100644 freqtrade/optimize/recursive_analysis.py create mode 100644 freqtrade/optimize/recursive_analysis_helpers.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index b9346fd5f..98e7cb084 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -20,7 +20,8 @@ from freqtrade.commands.list_commands import (start_list_exchanges, start_list_f start_list_timeframes, start_show_trades) from freqtrade.commands.optimize_commands import (start_backtesting, start_backtesting_show, start_edge, start_hyperopt, - start_lookahead_analysis) + start_lookahead_analysis, + start_recursive_analysis) from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit from freqtrade.commands.strategy_utils_commands import start_strategy_update diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 5473e95e1..70b6b9c01 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -122,6 +122,8 @@ ARGS_LOOKAHEAD_ANALYSIS = [ a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", 'cache') ] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"] +ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"] + class Arguments: """ @@ -206,7 +208,7 @@ class Arguments: start_list_strategies, start_list_timeframes, start_lookahead_analysis, start_new_config, start_new_strategy, start_plot_dataframe, start_plot_profit, - start_show_trades, start_strategy_update, + start_recursive_analysis, start_show_trades, start_strategy_update, start_test_pairlist, start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', @@ -467,3 +469,14 @@ class Arguments: self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS, parser=lookahead_analayis_cmd) + + # Add recursive_analysis subcommand + recursive_analayis_cmd = subparsers.add_parser( + 'recursive-analysis', + help="Check for potential look ahead bias.", + parents=[_common_parser, _strategy_parser]) + + recursive_analayis_cmd.set_defaults(func=start_recursive_analysis) + + self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS, + parser=recursive_analayis_cmd) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index cdddf0fe5..0a63753d9 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -144,3 +144,15 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) LookaheadAnalysisSubFunctions.start(config) + + +def start_recursive_analysis(args: Dict[str, Any]) -> None: + """ + Start the backtest recursive tester script + :param args: Cli args from Arguments() + :return: None + """ + from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + RecursiveAnalysisSubFunctions.start(config) diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py new file mode 100644 index 000000000..39aee28b3 --- /dev/null +++ b/freqtrade/optimize/recursive_analysis.py @@ -0,0 +1,236 @@ +import logging +import shutil +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from pandas import DataFrame + +from freqtrade.configuration import TimeRange +from freqtrade.data.history import get_timerange +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, + restore_verbosity_for_bias_tester) +from freqtrade.optimize.backtesting import Backtesting + + +logger = logging.getLogger(__name__) + + +class VarHolder: + timerange: TimeRange + data: DataFrame + indicators: Dict[str, DataFrame] + from_dt: datetime + to_dt: datetime + timeframe: str + startup_candle: int + +class RecursiveAnalysis: + + def __init__(self, config: Dict[str, Any], strategy_obj: Dict): + self.failed_bias_check = True + self.full_varHolder = VarHolder() + self.partial_varHolder_array = [] + self.partial_varHolder_lookahead_array = [] + + self.entry_varHolders: List[VarHolder] = [] + self.exit_varHolders: List[VarHolder] = [] + self.exchange: Optional[Any] = None + + # pull variables the scope of the recursive_analysis-instance + self.local_config = deepcopy(config) + self.local_config['strategy'] = strategy_obj['name'] + self._startup_candle = config.get('startup_candle', [199, 399, 499, 999, 1999]) + self.strategy_obj = strategy_obj + self.dict_recursive = dict() + + @staticmethod + def dt_to_timestamp(dt: datetime): + timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) + return timestamp + + # For recursive bias check + # analyzes two data frames with processed indicators and shows differences between them. + def analyze_indicators(self): + + pair_to_check = self.local_config['pairs'][0] + logger.info(f"Start checking for recursive bias") + + # check and report signals + base_last_row = self.full_varHolder.indicators[pair_to_check].iloc[-1] + base_timerange = self.full_varHolder.from_dt.strftime('%Y-%m-%dT%H:%M:%S') + "-" + self.full_varHolder.to_dt.strftime('%Y-%m-%dT%H:%M:%S') + + for part in self.partial_varHolder_array: + part_last_row = part.indicators[pair_to_check].iloc[-1] + part_timerange = part.from_dt.strftime('%Y-%m-%dT%H:%M:%S') + "-" + part.to_dt.strftime('%Y-%m-%dT%H:%M:%S') + + logger.info(f"Comparing last row of {base_timerange} backtest") + logger.info(f"vs {part_timerange} with {part.startup_candle} startup candle") + + compare_df = base_last_row.compare(part_last_row) + if compare_df.shape[0] > 0: + # print(compare_df) + for col_name, values in compare_df.items(): + # print(col_name) + if 'other' == col_name: + continue + indicators = values.index + + for indicator in indicators: + if(indicator not in self.dict_recursive): + self.dict_recursive[indicator] = {} + + values_diff = compare_df.loc[indicator] + values_diff_self = values_diff.loc['self'] + values_diff_other = values_diff.loc['other'] + difference = (values_diff_other - values_diff_self) / values_diff_self * 100 + + self.dict_recursive[indicator][part.startup_candle] = "{:.3f}%".format(difference) + + # logger.info(f"=> found difference in indicator " + # f"{indicator}, with difference of " + # "{:.8f}%".format(difference)) + + else: + logger.info("No difference found. Stop the process.") + break + + # For lookahead bias check + # analyzes two data frames with processed indicators and shows differences between them. + def analyze_indicators_lookahead(self): + + pair_to_check = self.local_config['pairs'][0] + logger.info(f"Start checking for lookahead bias") + + # check and report signals + # base_last_row = self.full_varHolder.indicators[pair_to_check].iloc[-1] + # base_timerange = self.full_varHolder.from_dt.strftime('%Y-%m-%dT%H:%M:%S') + "-" + self.full_varHolder.to_dt.strftime('%Y-%m-%dT%H:%M:%S') + + part = self.partial_varHolder_lookahead_array[0] + part_last_row = part.indicators[pair_to_check].iloc[-1] + date_to_check = part_last_row['date'] + base_row_to_check = self.full_varHolder.indicators[pair_to_check].loc[(self.full_varHolder.indicators[pair_to_check]['date'] == date_to_check)].iloc[-1] + + check_time = part.to_dt.strftime('%Y-%m-%dT%H:%M:%S') + + logger.info(f"Check indicators at {check_time}") + # logger.info(f"vs {part_timerange} with {part.startup_candle} startup candle") + + compare_df = base_row_to_check.compare(part_last_row) + if compare_df.shape[0] > 0: + # print(compare_df) + for col_name, values in compare_df.items(): + # print(col_name) + if 'other' == col_name: + continue + indicators = values.index + + for indicator in indicators: + logger.info(f"=> found lookahead in indicator {indicator}") + # logger.info("base value {:.5f}".format(values_diff_self)) + # logger.info("part value {:.5f}".format(values_diff_other)) + + else: + logger.info("No lookahead bias found. Stop the process.") + + def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]): + + if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: + # purge previous data if the freqai model is defined + # (to be sure nothing is carried over from older backtests) + path_to_current_identifier = ( + Path(f"{self.local_config['user_data_dir']}/models/" + f"{self.local_config['freqai']['identifier']}").resolve()) + # remove folder and its contents + if Path.exists(path_to_current_identifier): + shutil.rmtree(path_to_current_identifier) + + prepare_data_config = deepcopy(self.local_config) + prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + + str(self.dt_to_timestamp(varholder.to_dt))) + prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load + + backtesting = Backtesting(prepare_data_config, self.exchange) + self.exchange = backtesting.exchange + backtesting._set_strategy(backtesting.strategylist[0]) + + varholder.data, varholder.timerange = backtesting.load_bt_data() + backtesting.load_bt_data_detail() + varholder.timeframe = backtesting.timeframe + + varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) + + def fill_full_varholder(self): + self.full_varHolder = VarHolder() + + # define datetime in human-readable format + parsed_timerange = TimeRange.parse_timerange(self.local_config['timerange']) + + if parsed_timerange.startdt is None: + self.full_varHolder.from_dt = datetime.fromtimestamp(0, tz=timezone.utc) + else: + self.full_varHolder.from_dt = parsed_timerange.startdt + + if parsed_timerange.stopdt is None: + self.full_varHolder.to_dt = datetime.utcnow() + else: + self.full_varHolder.to_dt = parsed_timerange.stopdt + + self.prepare_data(self.full_varHolder, self.local_config['pairs']) + + def fill_partial_varholder(self, start_date, startup_candle): + partial_varHolder = VarHolder() + + partial_varHolder.from_dt = start_date + partial_varHolder.to_dt = self.full_varHolder.to_dt + partial_varHolder.startup_candle = startup_candle + + self.local_config['startup_candle_count'] = startup_candle + + self.prepare_data(partial_varHolder, self.local_config['pairs']) + + self.partial_varHolder_array.append(partial_varHolder) + + def fill_partial_varholder_lookahead(self, end_date): + partial_varHolder = VarHolder() + + partial_varHolder.from_dt = self.full_varHolder.from_dt + partial_varHolder.to_dt = end_date + # partial_varHolder.startup_candle = startup_candle + + # self.local_config['startup_candle_count'] = startup_candle + + self.prepare_data(partial_varHolder, self.local_config['pairs']) + + self.partial_varHolder_lookahead_array.append(partial_varHolder) + + def start(self) -> None: + + # first make a single backtest + self.fill_full_varholder() + + reduce_verbosity_for_bias_tester() + + start_date_full = self.full_varHolder.from_dt + end_date_full = self.full_varHolder.to_dt + + timeframe_minutes = timeframe_to_minutes(self.full_varHolder.timeframe) + + end_date_partial = start_date_full + timedelta(minutes=int(timeframe_minutes * 10)) + + self.fill_partial_varholder_lookahead(end_date_partial) + + # restore_verbosity_for_bias_tester() + + start_date_partial = end_date_full - timedelta(minutes=int(timeframe_minutes)) + + for startup_candle in self._startup_candle: + self.fill_partial_varholder(start_date_partial, int(startup_candle)) + + # Restore verbosity, so it's not too quiet for the next strategy + restore_verbosity_for_bias_tester() + + self.analyze_indicators() + self.analyze_indicators_lookahead() \ No newline at end of file diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py new file mode 100644 index 000000000..41ec31ae5 --- /dev/null +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -0,0 +1,182 @@ +import logging +import time +from pathlib import Path +from typing import Any, Dict, List + +import pandas as pd + +from freqtrade.constants import Config +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.recursive_analysis import RecursiveAnalysis +from freqtrade.resolvers import StrategyResolver + + +logger = logging.getLogger(__name__) + + +class RecursiveAnalysisSubFunctions: + + @staticmethod + def text_table_recursive_analysis_instances( + config: Dict[str, Any], + recursive_instances: List[RecursiveAnalysis]): + startups = recursive_instances[0]._startup_candle + headers = ['strategy', 'indicators'] + for candle in startups: + headers.append(candle) + + data = [] + for inst in recursive_instances: + if len(inst.dict_recursive) > 0: + for indicator, values in inst.dict_recursive.items(): + temp_data = [inst.strategy_obj['name'], indicator] + for candle in startups: + temp_data.append(values.get(int(candle), '-')) + data.append(temp_data) + + from tabulate import tabulate + table = tabulate(data, headers=headers, tablefmt="orgtbl") + print(table) + return table, headers, data + + @staticmethod + def export_to_csv(config: Dict[str, Any], lookahead_analysis: List[RecursiveAnalysis]): + def add_or_update_row(df, row_data): + if ( + (df['filename'] == row_data['filename']) & + (df['strategy'] == row_data['strategy']) + ).any(): + # Update existing row + pd_series = pd.DataFrame([row_data]) + df.loc[ + (df['filename'] == row_data['filename']) & + (df['strategy'] == row_data['strategy']) + ] = pd_series + else: + # Add new row + df = pd.concat([df, pd.DataFrame([row_data], columns=df.columns)]) + + return df + + if Path(config['lookahead_analysis_exportfilename']).exists(): + # Read CSV file into a pandas dataframe + csv_df = pd.read_csv(config['lookahead_analysis_exportfilename']) + else: + # Create a new empty DataFrame with the desired column names and set the index + csv_df = pd.DataFrame(columns=[ + 'filename', 'strategy', 'has_bias', 'total_signals', + 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators' + ], + index=None) + + for inst in lookahead_analysis: + # only update if + if (inst.current_analysis.total_signals > config['minimum_trade_amount'] + and inst.failed_bias_check is not True): + new_row_data = {'filename': inst.strategy_obj['location'].parts[-1], + 'strategy': inst.strategy_obj['name'], + 'has_bias': inst.current_analysis.has_bias, + 'total_signals': + int(inst.current_analysis.total_signals), + 'biased_entry_signals': + int(inst.current_analysis.false_entry_signals), + 'biased_exit_signals': + int(inst.current_analysis.false_exit_signals), + 'biased_indicators': + ",".join(inst.current_analysis.false_indicators)} + csv_df = add_or_update_row(csv_df, new_row_data) + + # Fill NaN values with a default value (e.g., 0) + csv_df['total_signals'] = csv_df['total_signals'].fillna(0) + csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].fillna(0) + csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].fillna(0) + + # Convert columns to integers + csv_df['total_signals'] = csv_df['total_signals'].astype(int) + csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].astype(int) + csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].astype(int) + + logger.info(f"saving {config['lookahead_analysis_exportfilename']}") + csv_df.to_csv(config['lookahead_analysis_exportfilename'], index=False) + + @staticmethod + def calculate_config_overrides(config: Config): + if config['targeted_trade_amount'] < config['minimum_trade_amount']: + # this combo doesn't make any sense. + raise OperationalException( + "Targeted trade amount can't be smaller than minimum trade amount." + ) + if len(config['pairs']) > config['max_open_trades']: + logger.info('Max_open_trades were less than amount of pairs. ' + 'Set max_open_trades to amount of pairs just to avoid false positives.') + config['max_open_trades'] = len(config['pairs']) + + min_dry_run_wallet = 1000000000 + if config['dry_run_wallet'] < min_dry_run_wallet: + logger.info('Dry run wallet was not set to 1 billion, pushing it up there ' + 'just to avoid false positives') + config['dry_run_wallet'] = min_dry_run_wallet + + # enforce cache to be 'none', shift it to 'none' if not already + # (since the default value is 'day') + if config.get('backtest_cache') is None: + config['backtest_cache'] = 'none' + elif config['backtest_cache'] != 'none': + logger.info(f"backtest_cache = " + f"{config['backtest_cache']} detected. " + f"Inside lookahead-analysis it is enforced to be 'none'. " + f"Changed it to 'none'") + config['backtest_cache'] = 'none' + return config + + @staticmethod + def initialize_single_recursive_analysis(config: Config, strategy_obj: Dict[str, Any]): + + logger.info(f"Recursive test of {Path(strategy_obj['location']).name} started.") + start = time.perf_counter() + current_instance = RecursiveAnalysis(config, strategy_obj) + current_instance.start() + elapsed = time.perf_counter() - start + logger.info(f"Checking recursive and lookahead bias of indicators " + f"of {Path(strategy_obj['location']).name} " + f"took {elapsed:.0f} seconds.") + return current_instance + + @staticmethod + def start(config: Config): + config = RecursiveAnalysisSubFunctions.calculate_config_overrides(config) + + strategy_objs = StrategyResolver.search_all_objects( + config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) + + RecursiveAnalysis_instances = [] + + # unify --strategy and --strategy_list to one list + if not (strategy_list := config.get('strategy_list', [])): + if config.get('strategy') is None: + raise OperationalException( + "No Strategy specified. Please specify a strategy via --strategy or " + "--strategy_list" + ) + strategy_list = [config['strategy']] + + # check if strategies can be properly loaded, only check them if they can be. + for strat in strategy_list: + for strategy_obj in strategy_objs: + if strategy_obj['name'] == strat and strategy_obj not in strategy_list: + RecursiveAnalysis_instances.append( + RecursiveAnalysisSubFunctions.initialize_single_recursive_analysis( + config, strategy_obj)) + break + + # report the results + if RecursiveAnalysis_instances: + RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances( + config, RecursiveAnalysis_instances) + if config.get('lookahead_analysis_exportfilename') is not None: + RecursiveAnalysisSubFunctions.export_to_csv(config, RecursiveAnalysis_instances) + else: + logger.error("There were no strategies specified neither through " + "--strategy nor through " + "--strategy_list " + "or timeframe was not specified.") From 607c604a45b81764401a988cc203d3733e9717fc Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 4 Sep 2023 11:16:10 +0900 Subject: [PATCH 02/52] add mising const --- freqtrade/constants.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 318c414e1..ecd9f6e5d 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -177,6 +177,11 @@ CONF_SCHEMA = { 'minimum_trade_amount': {'type': 'number', 'default': 10}, 'targeted_trade_amount': {'type': 'number', 'default': 20}, 'lookahead_analysis_exportfilename': {'type': 'string'}, + 'startup_candle': { + 'type': 'array', + 'uniqueItems': True, + 'default': [199, 399, 499, 999, 1999], + }, 'liquidation_buffer': {'type': 'number', 'minimum': 0.0, 'maximum': 0.99}, 'backtest_breakdown': { 'type': 'array', From feab5f82c15370596b9a2701248797d551e95c87 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 4 Sep 2023 11:20:49 +0900 Subject: [PATCH 03/52] add missing arg --- freqtrade/commands/cli_options.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 586318e30..dcbd9714a 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -705,4 +705,9 @@ AVAILABLE_CLI_OPTIONS = { help="Use this csv-filename to store lookahead-analysis-results", type=str ), + "startup_candle": Arg( + '--startup-candle', + help='Specify startup candles to be checked (`199`, `499`, `999`, `1999`).', + nargs='+', + ), } From 821a598ff4e58f2d38ea1e99f6e94bce3b18dd14 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 4 Sep 2023 11:35:44 +0900 Subject: [PATCH 04/52] fix ruff --- freqtrade/commands/arguments.py | 5 +-- freqtrade/optimize/recursive_analysis.py | 39 ++++++++---------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 70b6b9c01..2d82d8508 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -208,8 +208,9 @@ class Arguments: start_list_strategies, start_list_timeframes, start_lookahead_analysis, start_new_config, start_new_strategy, start_plot_dataframe, start_plot_profit, - start_recursive_analysis, start_show_trades, start_strategy_update, - start_test_pairlist, start_trading, start_webserver) + start_recursive_analysis, start_show_trades, + start_strategy_update, start_test_pairlist, + start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index 39aee28b3..d01c58758 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -8,7 +8,6 @@ from typing import Any, Dict, List, Optional from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.data.history import get_timerange from freqtrade.exchange import timeframe_to_minutes from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, restore_verbosity_for_bias_tester) @@ -54,21 +53,16 @@ class RecursiveAnalysis: # For recursive bias check # analyzes two data frames with processed indicators and shows differences between them. def analyze_indicators(self): - + pair_to_check = self.local_config['pairs'][0] - logger.info(f"Start checking for recursive bias") + logger.info("Start checking for recursive bias") # check and report signals base_last_row = self.full_varHolder.indicators[pair_to_check].iloc[-1] - base_timerange = self.full_varHolder.from_dt.strftime('%Y-%m-%dT%H:%M:%S') + "-" + self.full_varHolder.to_dt.strftime('%Y-%m-%dT%H:%M:%S') for part in self.partial_varHolder_array: part_last_row = part.indicators[pair_to_check].iloc[-1] - part_timerange = part.from_dt.strftime('%Y-%m-%dT%H:%M:%S') + "-" + part.to_dt.strftime('%Y-%m-%dT%H:%M:%S') - logger.info(f"Comparing last row of {base_timerange} backtest") - logger.info(f"vs {part_timerange} with {part.startup_candle} startup candle") - compare_df = base_last_row.compare(part_last_row) if compare_df.shape[0] > 0: # print(compare_df) @@ -85,13 +79,9 @@ class RecursiveAnalysis: values_diff = compare_df.loc[indicator] values_diff_self = values_diff.loc['self'] values_diff_other = values_diff.loc['other'] - difference = (values_diff_other - values_diff_self) / values_diff_self * 100 - - self.dict_recursive[indicator][part.startup_candle] = "{:.3f}%".format(difference) + diff = (values_diff_other - values_diff_self) / values_diff_self * 100 - # logger.info(f"=> found difference in indicator " - # f"{indicator}, with difference of " - # "{:.8f}%".format(difference)) + self.dict_recursive[indicator][part.startup_candle] = "{:.3f}%".format(diff) else: logger.info("No difference found. Stop the process.") @@ -100,25 +90,22 @@ class RecursiveAnalysis: # For lookahead bias check # analyzes two data frames with processed indicators and shows differences between them. def analyze_indicators_lookahead(self): - - pair_to_check = self.local_config['pairs'][0] - logger.info(f"Start checking for lookahead bias") - # check and report signals - # base_last_row = self.full_varHolder.indicators[pair_to_check].iloc[-1] - # base_timerange = self.full_varHolder.from_dt.strftime('%Y-%m-%dT%H:%M:%S') + "-" + self.full_varHolder.to_dt.strftime('%Y-%m-%dT%H:%M:%S') - + pair_to_check = self.local_config['pairs'][0] + logger.info("Start checking for lookahead bias on indicators only") + part = self.partial_varHolder_lookahead_array[0] part_last_row = part.indicators[pair_to_check].iloc[-1] date_to_check = part_last_row['date'] - base_row_to_check = self.full_varHolder.indicators[pair_to_check].loc[(self.full_varHolder.indicators[pair_to_check]['date'] == date_to_check)].iloc[-1] + index_to_get = (self.full_varHolder.indicators[pair_to_check]['date'] == date_to_check) + base_row_check = self.full_varHolder.indicators[pair_to_check].loc[index_to_get].iloc[-1] check_time = part.to_dt.strftime('%Y-%m-%dT%H:%M:%S') logger.info(f"Check indicators at {check_time}") # logger.info(f"vs {part_timerange} with {part.startup_candle} startup candle") - - compare_df = base_row_to_check.compare(part_last_row) + + compare_df = base_row_check.compare(part_last_row) if compare_df.shape[0] > 0: # print(compare_df) for col_name, values in compare_df.items(): @@ -133,7 +120,7 @@ class RecursiveAnalysis: # logger.info("part value {:.5f}".format(values_diff_other)) else: - logger.info("No lookahead bias found. Stop the process.") + logger.info("No lookahead bias on indicators found. Stop the process.") def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]): @@ -233,4 +220,4 @@ class RecursiveAnalysis: restore_verbosity_for_bias_tester() self.analyze_indicators() - self.analyze_indicators_lookahead() \ No newline at end of file + self.analyze_indicators_lookahead() From 3fea2a35a28bc3bf5a7280ec482a403612cae4a3 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 4 Sep 2023 11:38:13 +0900 Subject: [PATCH 05/52] pre-commit fixes --- freqtrade/commands/arguments.py | 4 ++-- freqtrade/optimize/recursive_analysis.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 2d82d8508..f7c17b09c 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -209,8 +209,8 @@ class Arguments: start_lookahead_analysis, start_new_config, start_new_strategy, start_plot_dataframe, start_plot_profit, start_recursive_analysis, start_show_trades, - start_strategy_update, start_test_pairlist, - start_trading, start_webserver) + start_strategy_update, start_test_pairlist, start_trading, + start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index d01c58758..4c6bc372e 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -59,7 +59,7 @@ class RecursiveAnalysis: # check and report signals base_last_row = self.full_varHolder.indicators[pair_to_check].iloc[-1] - + for part in self.partial_varHolder_array: part_last_row = part.indicators[pair_to_check].iloc[-1] From e1b6b9b5a666fe914999f34eb480c1af2496d661 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 4 Sep 2023 11:41:24 +0900 Subject: [PATCH 06/52] ruff fix --- freqtrade/optimize/recursive_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index 4c6bc372e..0f1240df2 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -81,7 +81,7 @@ class RecursiveAnalysis: values_diff_other = values_diff.loc['other'] diff = (values_diff_other - values_diff_self) / values_diff_self * 100 - self.dict_recursive[indicator][part.startup_candle] = "{:.3f}%".format(diff) + self.dict_recursive[indicator][part.startup_candle] = f"{diff:.3f}%" else: logger.info("No difference found. Stop the process.") From 0ada2d939041214707676c129b1b92eb77e226db Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 4 Sep 2023 11:45:25 +0900 Subject: [PATCH 07/52] fix mypy --- freqtrade/optimize/recursive_analysis.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index 0f1240df2..07237b969 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -26,13 +26,14 @@ class VarHolder: timeframe: str startup_candle: int + class RecursiveAnalysis: def __init__(self, config: Dict[str, Any], strategy_obj: Dict): self.failed_bias_check = True self.full_varHolder = VarHolder() - self.partial_varHolder_array = [] - self.partial_varHolder_lookahead_array = [] + self.partial_varHolder_array: List[VarHolder] = [] + self.partial_varHolder_lookahead_array: List[VarHolder] = [] self.entry_varHolders: List[VarHolder] = [] self.exit_varHolders: List[VarHolder] = [] @@ -43,7 +44,7 @@ class RecursiveAnalysis: self.local_config['strategy'] = strategy_obj['name'] self._startup_candle = config.get('startup_candle', [199, 399, 499, 999, 1999]) self.strategy_obj = strategy_obj - self.dict_recursive = dict() + self.dict_recursive: Dict[Any] = dict() @staticmethod def dt_to_timestamp(dt: datetime): @@ -73,7 +74,7 @@ class RecursiveAnalysis: indicators = values.index for indicator in indicators: - if(indicator not in self.dict_recursive): + if (indicator not in self.dict_recursive): self.dict_recursive[indicator] = {} values_diff = compare_df.loc[indicator] From a9f63c6a99adbd8cd71113aeff7ab7cb2c22a267 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 4 Sep 2023 11:52:09 +0900 Subject: [PATCH 08/52] fix mypy --- freqtrade/optimize/recursive_analysis.py | 2 +- .../optimize/recursive_analysis_helpers.py | 80 ------------------- 2 files changed, 1 insertion(+), 81 deletions(-) diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index 07237b969..ae91d49d2 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -44,7 +44,7 @@ class RecursiveAnalysis: self.local_config['strategy'] = strategy_obj['name'] self._startup_candle = config.get('startup_candle', [199, 399, 499, 999, 1999]) self.strategy_obj = strategy_obj - self.dict_recursive: Dict[Any] = dict() + self.dict_recursive: Dict[str, Any] = dict() @staticmethod def dt_to_timestamp(dt: datetime): diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py index 41ec31ae5..3b1a6d3e0 100644 --- a/freqtrade/optimize/recursive_analysis_helpers.py +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -39,86 +39,8 @@ class RecursiveAnalysisSubFunctions: print(table) return table, headers, data - @staticmethod - def export_to_csv(config: Dict[str, Any], lookahead_analysis: List[RecursiveAnalysis]): - def add_or_update_row(df, row_data): - if ( - (df['filename'] == row_data['filename']) & - (df['strategy'] == row_data['strategy']) - ).any(): - # Update existing row - pd_series = pd.DataFrame([row_data]) - df.loc[ - (df['filename'] == row_data['filename']) & - (df['strategy'] == row_data['strategy']) - ] = pd_series - else: - # Add new row - df = pd.concat([df, pd.DataFrame([row_data], columns=df.columns)]) - - return df - - if Path(config['lookahead_analysis_exportfilename']).exists(): - # Read CSV file into a pandas dataframe - csv_df = pd.read_csv(config['lookahead_analysis_exportfilename']) - else: - # Create a new empty DataFrame with the desired column names and set the index - csv_df = pd.DataFrame(columns=[ - 'filename', 'strategy', 'has_bias', 'total_signals', - 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators' - ], - index=None) - - for inst in lookahead_analysis: - # only update if - if (inst.current_analysis.total_signals > config['minimum_trade_amount'] - and inst.failed_bias_check is not True): - new_row_data = {'filename': inst.strategy_obj['location'].parts[-1], - 'strategy': inst.strategy_obj['name'], - 'has_bias': inst.current_analysis.has_bias, - 'total_signals': - int(inst.current_analysis.total_signals), - 'biased_entry_signals': - int(inst.current_analysis.false_entry_signals), - 'biased_exit_signals': - int(inst.current_analysis.false_exit_signals), - 'biased_indicators': - ",".join(inst.current_analysis.false_indicators)} - csv_df = add_or_update_row(csv_df, new_row_data) - - # Fill NaN values with a default value (e.g., 0) - csv_df['total_signals'] = csv_df['total_signals'].fillna(0) - csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].fillna(0) - csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].fillna(0) - - # Convert columns to integers - csv_df['total_signals'] = csv_df['total_signals'].astype(int) - csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].astype(int) - csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].astype(int) - - logger.info(f"saving {config['lookahead_analysis_exportfilename']}") - csv_df.to_csv(config['lookahead_analysis_exportfilename'], index=False) - @staticmethod def calculate_config_overrides(config: Config): - if config['targeted_trade_amount'] < config['minimum_trade_amount']: - # this combo doesn't make any sense. - raise OperationalException( - "Targeted trade amount can't be smaller than minimum trade amount." - ) - if len(config['pairs']) > config['max_open_trades']: - logger.info('Max_open_trades were less than amount of pairs. ' - 'Set max_open_trades to amount of pairs just to avoid false positives.') - config['max_open_trades'] = len(config['pairs']) - - min_dry_run_wallet = 1000000000 - if config['dry_run_wallet'] < min_dry_run_wallet: - logger.info('Dry run wallet was not set to 1 billion, pushing it up there ' - 'just to avoid false positives') - config['dry_run_wallet'] = min_dry_run_wallet - - # enforce cache to be 'none', shift it to 'none' if not already - # (since the default value is 'day') if config.get('backtest_cache') is None: config['backtest_cache'] = 'none' elif config['backtest_cache'] != 'none': @@ -173,8 +95,6 @@ class RecursiveAnalysisSubFunctions: if RecursiveAnalysis_instances: RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances( config, RecursiveAnalysis_instances) - if config.get('lookahead_analysis_exportfilename') is not None: - RecursiveAnalysisSubFunctions.export_to_csv(config, RecursiveAnalysis_instances) else: logger.error("There were no strategies specified neither through " "--strategy nor through " From cea3f7d3fa216bcea65e622a862e692bc27595a5 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 4 Sep 2023 11:53:59 +0900 Subject: [PATCH 09/52] fix flake8 --- freqtrade/optimize/recursive_analysis_helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py index 3b1a6d3e0..0b353edfc 100644 --- a/freqtrade/optimize/recursive_analysis_helpers.py +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -3,8 +3,6 @@ import time from pathlib import Path from typing import Any, Dict, List -import pandas as pd - from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.optimize.recursive_analysis import RecursiveAnalysis From 008f62121137a5793d2a3e58de5c4b55f16a6a71 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 12 Sep 2023 15:42:32 +0900 Subject: [PATCH 10/52] create BaseAnalysis class --- freqtrade/optimize/base_analysis.py | 104 +++++++++++++++++++++++ freqtrade/optimize/recursive_analysis.py | 84 ++---------------- 2 files changed, 109 insertions(+), 79 deletions(-) create mode 100644 freqtrade/optimize/base_analysis.py diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py new file mode 100644 index 000000000..43beaeb1e --- /dev/null +++ b/freqtrade/optimize/base_analysis.py @@ -0,0 +1,104 @@ +import logging +import shutil +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from pandas import DataFrame + +from freqtrade.configuration import TimeRange +from freqtrade.data.history import get_timerange +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, + restore_verbosity_for_bias_tester) +from freqtrade.optimize.backtesting import Backtesting + + +logger = logging.getLogger(__name__) + + +class VarHolder: + timerange: TimeRange + data: DataFrame + indicators: Dict[str, DataFrame] + result: DataFrame + compared: DataFrame + from_dt: datetime + to_dt: datetime + compared_dt: datetime + timeframe: str + startup_candle: int + + +class BaseAnalysis: + + def __init__(self, config: Dict[str, Any], strategy_obj: Dict): + self.failed_bias_check = True + self.full_varHolder = VarHolder() + self._fee = None + + # pull variables the scope of the lookahead_analysis-instance + self.local_config = deepcopy(config) + self.local_config['strategy'] = strategy_obj['name'] + self.strategy_obj = strategy_obj + + @staticmethod + def dt_to_timestamp(dt: datetime): + timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) + return timestamp + + def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]): + + if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: + # purge previous data if the freqai model is defined + # (to be sure nothing is carried over from older backtests) + path_to_current_identifier = ( + Path(f"{self.local_config['user_data_dir']}/models/" + f"{self.local_config['freqai']['identifier']}").resolve()) + # remove folder and its contents + if Path.exists(path_to_current_identifier): + shutil.rmtree(path_to_current_identifier) + + prepare_data_config = deepcopy(self.local_config) + prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + + str(self.dt_to_timestamp(varholder.to_dt))) + prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load + + if self._fee is not None: + # Don't re-calculate fee per pair, as fee might differ per pair. + prepare_data_config['fee'] = self._fee + + backtesting = Backtesting(prepare_data_config, self.exchange) + backtesting._set_strategy(backtesting.strategylist[0]) + + varholder.data, varholder.timerange = backtesting.load_bt_data() + backtesting.load_bt_data_detail() + varholder.timeframe = backtesting.timeframe + + varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) + + def fill_full_varholder(self): + self.full_varHolder = VarHolder() + + # define datetime in human-readable format + parsed_timerange = TimeRange.parse_timerange(self.local_config['timerange']) + + if parsed_timerange.startdt is None: + self.full_varHolder.from_dt = datetime.fromtimestamp(0, tz=timezone.utc) + else: + self.full_varHolder.from_dt = parsed_timerange.startdt + + if parsed_timerange.stopdt is None: + self.full_varHolder.to_dt = datetime.utcnow() + else: + self.full_varHolder.to_dt = parsed_timerange.stopdt + + self.prepare_data(self.full_varHolder, self.local_config['pairs']) + + def start(self) -> None: + + # first make a single backtest + self.fill_full_varholder() + + reduce_verbosity_for_bias_tester() diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index ae91d49d2..6012760cf 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -12,45 +12,22 @@ from freqtrade.exchange import timeframe_to_minutes from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, restore_verbosity_for_bias_tester) from freqtrade.optimize.backtesting import Backtesting +from freqtrade.optimize.base_analysis import BaseAnalysis, VarHolder logger = logging.getLogger(__name__) -class VarHolder: - timerange: TimeRange - data: DataFrame - indicators: Dict[str, DataFrame] - from_dt: datetime - to_dt: datetime - timeframe: str - startup_candle: int - - -class RecursiveAnalysis: +class RecursiveAnalysis(BaseAnalysis): def __init__(self, config: Dict[str, Any], strategy_obj: Dict): - self.failed_bias_check = True - self.full_varHolder = VarHolder() + super().__init__(config, strategy_obj) self.partial_varHolder_array: List[VarHolder] = [] self.partial_varHolder_lookahead_array: List[VarHolder] = [] - self.entry_varHolders: List[VarHolder] = [] - self.exit_varHolders: List[VarHolder] = [] - self.exchange: Optional[Any] = None - - # pull variables the scope of the recursive_analysis-instance - self.local_config = deepcopy(config) - self.local_config['strategy'] = strategy_obj['name'] self._startup_candle = config.get('startup_candle', [199, 399, 499, 999, 1999]) - self.strategy_obj = strategy_obj self.dict_recursive: Dict[str, Any] = dict() - @staticmethod - def dt_to_timestamp(dt: datetime): - timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) - return timestamp - # For recursive bias check # analyzes two data frames with processed indicators and shows differences between them. def analyze_indicators(self): @@ -123,51 +100,6 @@ class RecursiveAnalysis: else: logger.info("No lookahead bias on indicators found. Stop the process.") - def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]): - - if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: - # purge previous data if the freqai model is defined - # (to be sure nothing is carried over from older backtests) - path_to_current_identifier = ( - Path(f"{self.local_config['user_data_dir']}/models/" - f"{self.local_config['freqai']['identifier']}").resolve()) - # remove folder and its contents - if Path.exists(path_to_current_identifier): - shutil.rmtree(path_to_current_identifier) - - prepare_data_config = deepcopy(self.local_config) - prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + - str(self.dt_to_timestamp(varholder.to_dt))) - prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load - - backtesting = Backtesting(prepare_data_config, self.exchange) - self.exchange = backtesting.exchange - backtesting._set_strategy(backtesting.strategylist[0]) - - varholder.data, varholder.timerange = backtesting.load_bt_data() - backtesting.load_bt_data_detail() - varholder.timeframe = backtesting.timeframe - - varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) - - def fill_full_varholder(self): - self.full_varHolder = VarHolder() - - # define datetime in human-readable format - parsed_timerange = TimeRange.parse_timerange(self.local_config['timerange']) - - if parsed_timerange.startdt is None: - self.full_varHolder.from_dt = datetime.fromtimestamp(0, tz=timezone.utc) - else: - self.full_varHolder.from_dt = parsed_timerange.startdt - - if parsed_timerange.stopdt is None: - self.full_varHolder.to_dt = datetime.utcnow() - else: - self.full_varHolder.to_dt = parsed_timerange.stopdt - - self.prepare_data(self.full_varHolder, self.local_config['pairs']) - def fill_partial_varholder(self, start_date, startup_candle): partial_varHolder = VarHolder() @@ -186,9 +118,6 @@ class RecursiveAnalysis: partial_varHolder.from_dt = self.full_varHolder.from_dt partial_varHolder.to_dt = end_date - # partial_varHolder.startup_candle = startup_candle - - # self.local_config['startup_candle_count'] = startup_candle self.prepare_data(partial_varHolder, self.local_config['pairs']) @@ -196,11 +125,8 @@ class RecursiveAnalysis: def start(self) -> None: - # first make a single backtest - self.fill_full_varholder() - - reduce_verbosity_for_bias_tester() - + super().start() + start_date_full = self.full_varHolder.from_dt end_date_full = self.full_varHolder.to_dt From 40695a39d53d12968ecafaef09d0de40810ec57b Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 12 Sep 2023 16:14:25 +0900 Subject: [PATCH 11/52] add missing var --- freqtrade/optimize/base_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py index 43beaeb1e..d435aead2 100644 --- a/freqtrade/optimize/base_analysis.py +++ b/freqtrade/optimize/base_analysis.py @@ -36,6 +36,7 @@ class BaseAnalysis: def __init__(self, config: Dict[str, Any], strategy_obj: Dict): self.failed_bias_check = True self.full_varHolder = VarHolder() + self.exchange: Optional[Any] = None self._fee = None # pull variables the scope of the lookahead_analysis-instance From 6360e7fb152c7c99d476b2d00dc941cbeacc73a4 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 12 Sep 2023 16:20:04 +0900 Subject: [PATCH 12/52] debug --- freqtrade/optimize/base_analysis.py | 2 -- freqtrade/optimize/recursive_analysis.py | 9 +++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py index d435aead2..ab7e6af0a 100644 --- a/freqtrade/optimize/base_analysis.py +++ b/freqtrade/optimize/base_analysis.py @@ -101,5 +101,3 @@ class BaseAnalysis: # first make a single backtest self.fill_full_varholder() - - reduce_verbosity_for_bias_tester() diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index 6012760cf..ef88f6927 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -21,11 +21,15 @@ logger = logging.getLogger(__name__) class RecursiveAnalysis(BaseAnalysis): def __init__(self, config: Dict[str, Any], strategy_obj: Dict): + + self._startup_candle = config.get('startup_candle', [199, 399, 499, 999, 1999]) + super().__init__(config, strategy_obj) + self.partial_varHolder_array: List[VarHolder] = [] self.partial_varHolder_lookahead_array: List[VarHolder] = [] - self._startup_candle = config.get('startup_candle', [199, 399, 499, 999, 1999]) + self.dict_recursive: Dict[str, Any] = dict() # For recursive bias check @@ -126,7 +130,8 @@ class RecursiveAnalysis(BaseAnalysis): def start(self) -> None: super().start() - + + reduce_verbosity_for_bias_tester() start_date_full = self.full_varHolder.from_dt end_date_full = self.full_varHolder.to_dt From 05f0dccb8eb184da48debd54bcce02349144ed02 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 12 Sep 2023 16:25:25 +0900 Subject: [PATCH 13/52] add missing args to config --- freqtrade/configuration/configuration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 43ede568c..a92814cf7 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -490,6 +490,9 @@ class Configuration: self._args_to_config(config, argname='lookahead_analysis_exportfilename', logstring='Path to store lookahead-analysis-results: {}') + self._args_to_config(config, argname='startup_candle', + logstring='Startup candle to be used on recursive analysis: {}') + def _process_runmode(self, config: Config) -> None: self._args_to_config(config, argname='dry_run', From 4d1810c2b669b4f73c71e0fa4b2cda23d60016fa Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 12 Sep 2023 19:11:19 +0900 Subject: [PATCH 14/52] update lookahead analysis --- freqtrade/optimize/base_analysis.py | 5 +- freqtrade/optimize/lookahead_analysis.py | 87 ++++++++---------------- 2 files changed, 33 insertions(+), 59 deletions(-) diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py index ab7e6af0a..5ee5a4466 100644 --- a/freqtrade/optimize/base_analysis.py +++ b/freqtrade/optimize/base_analysis.py @@ -49,7 +49,7 @@ class BaseAnalysis: timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) return timestamp - def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]): + def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame], backtesting=None): if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: # purge previous data if the freqai model is defined @@ -70,7 +70,8 @@ class BaseAnalysis: # Don't re-calculate fee per pair, as fee might differ per pair. prepare_data_config['fee'] = self._fee - backtesting = Backtesting(prepare_data_config, self.exchange) + if backtesting is None: + backtesting = Backtesting(prepare_data_config, self.exchange) backtesting._set_strategy(backtesting.strategylist[0]) varholder.data, varholder.timerange = backtesting.load_bt_data() diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 80418da95..00283da91 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -13,6 +13,7 @@ from freqtrade.exchange import timeframe_to_minutes from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, restore_verbosity_for_bias_tester) from freqtrade.optimize.backtesting import Backtesting +from freqtrade.optimize.base_analysis import BaseAnalysis, VarHolder logger = logging.getLogger(__name__) @@ -39,29 +40,18 @@ class Analysis: self.has_bias = False -class LookaheadAnalysis: +class LookaheadAnalysis(BaseAnalysis): def __init__(self, config: Dict[str, Any], strategy_obj: Dict): - self.failed_bias_check = True - self.full_varHolder = VarHolder() + + super().__init__(config, strategy_obj) self.entry_varHolders: List[VarHolder] = [] self.exit_varHolders: List[VarHolder] = [] - self.exchange: Optional[Any] = None - self._fee = None - # pull variables the scope of the lookahead_analysis-instance - self.local_config = deepcopy(config) - self.local_config['strategy'] = strategy_obj['name'] self.current_analysis = Analysis() self.minimum_trade_amount = config['minimum_trade_amount'] self.targeted_trade_amount = config['targeted_trade_amount'] - self.strategy_obj = strategy_obj - - @staticmethod - def dt_to_timestamp(dt: datetime): - timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) - return timestamp @staticmethod def get_result(backtesting: Backtesting, processed: DataFrame): @@ -129,57 +119,41 @@ class LookaheadAnalysis: f"{col_name[0]}. " f"{str(self_value)} != {str(other_value)}") - def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]): + def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame], backtesting=None): - if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: - # purge previous data if the freqai model is defined - # (to be sure nothing is carried over from older backtests) - path_to_current_identifier = ( - Path(f"{self.local_config['user_data_dir']}/models/" - f"{self.local_config['freqai']['identifier']}").resolve()) - # remove folder and its contents - if Path.exists(path_to_current_identifier): - shutil.rmtree(path_to_current_identifier) + super().prepare_data(varholder, pairs_to_load, backtesting) - prepare_data_config = deepcopy(self.local_config) - prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + - str(self.dt_to_timestamp(varholder.to_dt))) - prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load + # if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: + # # purge previous data if the freqai model is defined + # # (to be sure nothing is carried over from older backtests) + # path_to_current_identifier = ( + # Path(f"{self.local_config['user_data_dir']}/models/" + # f"{self.local_config['freqai']['identifier']}").resolve()) + # # remove folder and its contents + # if Path.exists(path_to_current_identifier): + # shutil.rmtree(path_to_current_identifier) - if self._fee is not None: - # Don't re-calculate fee per pair, as fee might differ per pair. - prepare_data_config['fee'] = self._fee + # prepare_data_config = deepcopy(self.local_config) + # prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + + # str(self.dt_to_timestamp(varholder.to_dt))) + # prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load - backtesting = Backtesting(prepare_data_config, self.exchange) + # if self._fee is not None: + # # Don't re-calculate fee per pair, as fee might differ per pair. + # prepare_data_config['fee'] = self._fee + + # backtesting = Backtesting(prepare_data_config, self.exchange) self.exchange = backtesting.exchange self._fee = backtesting.fee - backtesting._set_strategy(backtesting.strategylist[0]) + # backtesting._set_strategy(backtesting.strategylist[0]) - varholder.data, varholder.timerange = backtesting.load_bt_data() - backtesting.load_bt_data_detail() - varholder.timeframe = backtesting.timeframe + # varholder.data, varholder.timerange = backtesting.load_bt_data() + # backtesting.load_bt_data_detail() + # varholder.timeframe = backtesting.timeframe - varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) + # varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) varholder.result = self.get_result(backtesting, varholder.indicators) - def fill_full_varholder(self): - self.full_varHolder = VarHolder() - - # define datetime in human-readable format - parsed_timerange = TimeRange.parse_timerange(self.local_config['timerange']) - - if parsed_timerange.startdt is None: - self.full_varHolder.from_dt = datetime.fromtimestamp(0, tz=timezone.utc) - else: - self.full_varHolder.from_dt = parsed_timerange.startdt - - if parsed_timerange.stopdt is None: - self.full_varHolder.to_dt = datetime.utcnow() - else: - self.full_varHolder.to_dt = parsed_timerange.stopdt - - self.prepare_data(self.full_varHolder, self.local_config['pairs']) - def fill_entry_and_exit_varHolders(self, result_row): # entry_varHolder entry_varHolder = VarHolder() @@ -246,8 +220,7 @@ class LookaheadAnalysis: def start(self) -> None: - # first make a single backtest - self.fill_full_varholder() + super().start() reduce_verbosity_for_bias_tester() From 475d8486bb7fbf8b6e128e8e549e6554a48ec72a Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 12 Sep 2023 19:21:01 +0900 Subject: [PATCH 15/52] fix mutable Backtest --- freqtrade/optimize/base_analysis.py | 5 ++--- freqtrade/optimize/lookahead_analysis.py | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py index 5ee5a4466..35c6b7462 100644 --- a/freqtrade/optimize/base_analysis.py +++ b/freqtrade/optimize/base_analysis.py @@ -49,7 +49,7 @@ class BaseAnalysis: timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) return timestamp - def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame], backtesting=None): + def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame], backtesting: Backtesting): if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: # purge previous data if the freqai model is defined @@ -70,8 +70,7 @@ class BaseAnalysis: # Don't re-calculate fee per pair, as fee might differ per pair. prepare_data_config['fee'] = self._fee - if backtesting is None: - backtesting = Backtesting(prepare_data_config, self.exchange) + backtesting = Backtesting(prepare_data_config, self.exchange) backtesting._set_strategy(backtesting.strategylist[0]) varholder.data, varholder.timerange = backtesting.load_bt_data() diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 00283da91..f15cf5553 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -121,6 +121,8 @@ class LookaheadAnalysis(BaseAnalysis): def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame], backtesting=None): + if backtesting is None: + backtesting = Backtesting(deepcopy(self.local_config)) super().prepare_data(varholder, pairs_to_load, backtesting) # if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: From cfeefa8754dcd2718d2473be9bee4b832e7d50d9 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 12 Sep 2023 19:29:13 +0900 Subject: [PATCH 16/52] remove prepare data from baseanalysis --- freqtrade/optimize/base_analysis.py | 30 -------------- freqtrade/optimize/lookahead_analysis.py | 50 +++++++++++------------- freqtrade/optimize/recursive_analysis.py | 26 ++++++++++++ 3 files changed, 49 insertions(+), 57 deletions(-) diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py index 35c6b7462..77d3523c0 100644 --- a/freqtrade/optimize/base_analysis.py +++ b/freqtrade/optimize/base_analysis.py @@ -49,36 +49,6 @@ class BaseAnalysis: timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) return timestamp - def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame], backtesting: Backtesting): - - if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: - # purge previous data if the freqai model is defined - # (to be sure nothing is carried over from older backtests) - path_to_current_identifier = ( - Path(f"{self.local_config['user_data_dir']}/models/" - f"{self.local_config['freqai']['identifier']}").resolve()) - # remove folder and its contents - if Path.exists(path_to_current_identifier): - shutil.rmtree(path_to_current_identifier) - - prepare_data_config = deepcopy(self.local_config) - prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + - str(self.dt_to_timestamp(varholder.to_dt))) - prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load - - if self._fee is not None: - # Don't re-calculate fee per pair, as fee might differ per pair. - prepare_data_config['fee'] = self._fee - - backtesting = Backtesting(prepare_data_config, self.exchange) - backtesting._set_strategy(backtesting.strategylist[0]) - - varholder.data, varholder.timerange = backtesting.load_bt_data() - backtesting.load_bt_data_detail() - varholder.timeframe = backtesting.timeframe - - varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) - def fill_full_varholder(self): self.full_varHolder = VarHolder() diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index f15cf5553..7b65099c8 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -119,41 +119,37 @@ class LookaheadAnalysis(BaseAnalysis): f"{col_name[0]}. " f"{str(self_value)} != {str(other_value)}") - def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame], backtesting=None): + def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]): - if backtesting is None: - backtesting = Backtesting(deepcopy(self.local_config)) - super().prepare_data(varholder, pairs_to_load, backtesting) + if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: + # purge previous data if the freqai model is defined + # (to be sure nothing is carried over from older backtests) + path_to_current_identifier = ( + Path(f"{self.local_config['user_data_dir']}/models/" + f"{self.local_config['freqai']['identifier']}").resolve()) + # remove folder and its contents + if Path.exists(path_to_current_identifier): + shutil.rmtree(path_to_current_identifier) - # if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: - # # purge previous data if the freqai model is defined - # # (to be sure nothing is carried over from older backtests) - # path_to_current_identifier = ( - # Path(f"{self.local_config['user_data_dir']}/models/" - # f"{self.local_config['freqai']['identifier']}").resolve()) - # # remove folder and its contents - # if Path.exists(path_to_current_identifier): - # shutil.rmtree(path_to_current_identifier) + prepare_data_config = deepcopy(self.local_config) + prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + + str(self.dt_to_timestamp(varholder.to_dt))) + prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load - # prepare_data_config = deepcopy(self.local_config) - # prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + - # str(self.dt_to_timestamp(varholder.to_dt))) - # prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load + if self._fee is not None: + # Don't re-calculate fee per pair, as fee might differ per pair. + prepare_data_config['fee'] = self._fee - # if self._fee is not None: - # # Don't re-calculate fee per pair, as fee might differ per pair. - # prepare_data_config['fee'] = self._fee - - # backtesting = Backtesting(prepare_data_config, self.exchange) + backtesting = Backtesting(prepare_data_config, self.exchange) self.exchange = backtesting.exchange self._fee = backtesting.fee - # backtesting._set_strategy(backtesting.strategylist[0]) + backtesting._set_strategy(backtesting.strategylist[0]) - # varholder.data, varholder.timerange = backtesting.load_bt_data() - # backtesting.load_bt_data_detail() - # varholder.timeframe = backtesting.timeframe + varholder.data, varholder.timerange = backtesting.load_bt_data() + backtesting.load_bt_data_detail() + varholder.timeframe = backtesting.timeframe - # varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) + varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) varholder.result = self.get_result(backtesting, varholder.indicators) def fill_entry_and_exit_varHolders(self, result_row): diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index ef88f6927..cacda3f17 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -104,6 +104,32 @@ class RecursiveAnalysis(BaseAnalysis): else: logger.info("No lookahead bias on indicators found. Stop the process.") + def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]): + + if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: + # purge previous data if the freqai model is defined + # (to be sure nothing is carried over from older backtests) + path_to_current_identifier = ( + Path(f"{self.local_config['user_data_dir']}/models/" + f"{self.local_config['freqai']['identifier']}").resolve()) + # remove folder and its contents + if Path.exists(path_to_current_identifier): + shutil.rmtree(path_to_current_identifier) + + prepare_data_config = deepcopy(self.local_config) + prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + + str(self.dt_to_timestamp(varholder.to_dt))) + prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load + + backtesting = Backtesting(prepare_data_config, self.exchange) + backtesting._set_strategy(backtesting.strategylist[0]) + + varholder.data, varholder.timerange = backtesting.load_bt_data() + backtesting.load_bt_data_detail() + varholder.timeframe = backtesting.timeframe + + varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) + def fill_partial_varholder(self, start_date, startup_candle): partial_varHolder = VarHolder() From bd9ea9bd8cdb3a98a126be8b0511c7045dd346d6 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 12 Sep 2023 19:50:39 +0900 Subject: [PATCH 17/52] precommit fix --- freqtrade/configuration/configuration.py | 2 +- freqtrade/optimize/recursive_analysis.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index a92814cf7..e5e4d28a0 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -492,7 +492,7 @@ class Configuration: self._args_to_config(config, argname='startup_candle', logstring='Startup candle to be used on recursive analysis: {}') - + def _process_runmode(self, config: Config) -> None: self._args_to_config(config, argname='dry_run', diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index cacda3f17..9bb326c7b 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -25,11 +25,11 @@ class RecursiveAnalysis(BaseAnalysis): self._startup_candle = config.get('startup_candle', [199, 399, 499, 999, 1999]) super().__init__(config, strategy_obj) - + self.partial_varHolder_array: List[VarHolder] = [] self.partial_varHolder_lookahead_array: List[VarHolder] = [] - + self.dict_recursive: Dict[str, Any] = dict() # For recursive bias check From 6377fd26892fd38e0bfcdd10c9867d382df4350d Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 12 Sep 2023 19:54:25 +0900 Subject: [PATCH 18/52] flake8 fix --- freqtrade/optimize/base_analysis.py | 11 ++--------- freqtrade/optimize/lookahead_analysis.py | 6 +++--- freqtrade/optimize/recursive_analysis.py | 5 ++--- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py index 77d3523c0..190ac882f 100644 --- a/freqtrade/optimize/base_analysis.py +++ b/freqtrade/optimize/base_analysis.py @@ -1,18 +1,11 @@ import logging -import shutil from copy import deepcopy -from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Any, Dict, List, Optional +from datetime import datetime, timezone +from typing import Any, Dict, Optional from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.data.history import get_timerange -from freqtrade.exchange import timeframe_to_minutes -from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, - restore_verbosity_for_bias_tester) -from freqtrade.optimize.backtesting import Backtesting logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 7b65099c8..b6c419bb7 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -1,9 +1,9 @@ import logging import shutil from copy import deepcopy -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List from pandas import DataFrame @@ -13,7 +13,7 @@ from freqtrade.exchange import timeframe_to_minutes from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, restore_verbosity_for_bias_tester) from freqtrade.optimize.backtesting import Backtesting -from freqtrade.optimize.base_analysis import BaseAnalysis, VarHolder +from freqtrade.optimize.base_analysis import BaseAnalysis logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index 9bb326c7b..dc069040b 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -1,13 +1,12 @@ import logging import shutil from copy import deepcopy -from datetime import datetime, timedelta, timezone +from datetime import timedelta from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List from pandas import DataFrame -from freqtrade.configuration import TimeRange from freqtrade.exchange import timeframe_to_minutes from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, restore_verbosity_for_bias_tester) From a0e0d7fe27fb0093e0c898f179e2286621a8bd99 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 12 Sep 2023 19:57:16 +0900 Subject: [PATCH 19/52] more fixes --- freqtrade/optimize/lookahead_analysis.py | 14 +------------- freqtrade/optimize/recursive_analysis.py | 1 - 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index b6c419bb7..8be057b2e 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -13,24 +13,12 @@ from freqtrade.exchange import timeframe_to_minutes from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, restore_verbosity_for_bias_tester) from freqtrade.optimize.backtesting import Backtesting -from freqtrade.optimize.base_analysis import BaseAnalysis +from freqtrade.optimize.base_analysis import BaseAnalysis, VarHolder logger = logging.getLogger(__name__) -class VarHolder: - timerange: TimeRange - data: DataFrame - indicators: Dict[str, DataFrame] - result: DataFrame - compared: DataFrame - from_dt: datetime - to_dt: datetime - compared_dt: datetime - timeframe: str - - class Analysis: def __init__(self) -> None: self.total_signals = 0 diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index dc069040b..599fc4dda 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -28,7 +28,6 @@ class RecursiveAnalysis(BaseAnalysis): self.partial_varHolder_array: List[VarHolder] = [] self.partial_varHolder_lookahead_array: List[VarHolder] = [] - self.dict_recursive: Dict[str, Any] = dict() # For recursive bias check From 5019fb5bf349f8f5beb353c7f9ab451386611147 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 12 Sep 2023 19:58:40 +0900 Subject: [PATCH 20/52] fix flake8 --- freqtrade/optimize/lookahead_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 8be057b2e..924e43e07 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -7,7 +7,6 @@ from typing import Any, Dict, List from pandas import DataFrame -from freqtrade.configuration import TimeRange from freqtrade.data.history import get_timerange from freqtrade.exchange import timeframe_to_minutes from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, From 08dffc95d877ebcb605835b186ce4924c409f209 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 13 Sep 2023 11:58:28 +0900 Subject: [PATCH 21/52] fix wording --- freqtrade/optimize/recursive_analysis_helpers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py index 0b353edfc..8bae36dd2 100644 --- a/freqtrade/optimize/recursive_analysis_helpers.py +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -39,12 +39,19 @@ class RecursiveAnalysisSubFunctions: @staticmethod def calculate_config_overrides(config: Config): + if 'timerange' not in config: + # setting a timerange is enforced here + raise OperationalException( + "Please set a timerange. " + "A timerange of 20 candles are enough for recursive analysis." + ) + if config.get('backtest_cache') is None: config['backtest_cache'] = 'none' elif config['backtest_cache'] != 'none': logger.info(f"backtest_cache = " f"{config['backtest_cache']} detected. " - f"Inside lookahead-analysis it is enforced to be 'none'. " + f"Inside recursive-analysis it is enforced to be 'none'. " f"Changed it to 'none'") config['backtest_cache'] = 'none' return config @@ -57,7 +64,7 @@ class RecursiveAnalysisSubFunctions: current_instance = RecursiveAnalysis(config, strategy_obj) current_instance.start() elapsed = time.perf_counter() - start - logger.info(f"Checking recursive and lookahead bias of indicators " + logger.info(f"Checking recursive and indicator-only lookahead bias of indicators " f"of {Path(strategy_obj['location']).name} " f"took {elapsed:.0f} seconds.") return current_instance From 979e485f24dfda6fba3e1a07950a6adc71a131b3 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 20 Sep 2023 20:53:34 +0900 Subject: [PATCH 22/52] initial doc --- docs/recursive-analysis.md | 96 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 docs/recursive-analysis.md diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md new file mode 100644 index 000000000..3a8b3017c --- /dev/null +++ b/docs/recursive-analysis.md @@ -0,0 +1,96 @@ +# Lookahead analysis + +This page explains how to validate your strategy in terms of recursive formula issue. + +First of all, what is recursive formula? Recursive formula is a formula that defines any term of a sequence in terms of its preceding term(s). Example of a recursive formula is an = an-1 + b. + +Second question is why is it matter for Freqtrade? It matters because in backtesting, the bot will get full data of the pairs according to the timerange specified. But in dry/live run, the bot will have limited amounts of data, limited by what each exchanges gives. + +For example, let's say that I want to calculate a very basic indicator called `steps`. The first row's value is always 0, while the following rows' values are equal to the value of the previous row's plus 1. If I were to calculate it using latest 1000 candles, then the `steps` value of first row is 0, and the `steps` value at last closed candle is 999. + +But what if I only calculate based of latest 500 candles? Then instead of 999, the `steps` value at last closed candle is 499. The difference of the value means your backtest result can differ from your dry/live run result. + +Recursive-analysis requires historic data to be available. To learn how to get data for the pairs and exchange you're interested in, +head over to the [Data Downloading](data-download.md) section of the documentation. + +This command is built upon backtesting since it internally chains backtests to prepare different lenghts of data and calculate indicators based of each of the prepared data. +This is done by not looking at the strategy itself - but at the value of the indicators it returned. After multiple backtests are done to calculate the indicators of different startup candles value, the values of last rows are compared to see hoe much differences are they compared to the base backtest. + +You can use commands of [Backtesting](backtesting.md). + +- `--cache` is forced to "none". +- Since we are only looking at indicators' value, using more than one pair is redundant. It is recommended to set the pair used in the command using `-p` flag, preferably using pair with high price, such as BTC or ETH, to avoid having rounding issue that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis the first pair in the whitelist. +- It's recommended to set a long timerange (at least consist of 5000 candles), so that the initial backtest that going to be used as benchmark have very small or no recursive issue at all. For example, for a 5m timeframe, timerange of 5000 candles would be equal to 18 days. + +Beside recursive formula check, this command also going to do a simple lookahead bias check on the indicators' value only. It won't replace [Lookahead-analysis](lookahead-analysis.md), since this check won't check the difference in trades' entries and exits. It will only check whether there is any difference in indicators' value if the end of the data are moved. + +## Recursive-analysis command reference + +``` +usage: freqtrade recursive-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [-s NAME] + [--strategy-path PATH] + [--recursive-strategy-search] + [--freqaimodel NAME] + [--freqaimodel-path PATH] [-i TIMEFRAME] + [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}] + [-p PAIRS [PAIRS ...]] + [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] + [--export {none,trades,signals}] + [--export-filename PATH] + [--breakdown {day,week,month} [{day,week,month} ...]] + [--cache {none,day,week,month}] + [--freqai-backtest-live-models] + [--minimum-trade-amount INT] + [--targeted-trade-amount INT] + [--lookahead-analysis-exportfilename LOOKAHEAD_ANALYSIS_EXPORTFILENAME] + +options: + --minimum-trade-amount INT + Minimum trade amount for lookahead-analysis + --targeted-trade-amount INT + Targeted trade amount for lookahead analysis + --lookahead-analysis-exportfilename LOOKAHEAD_ANALYSIS_EXPORTFILENAME + Use this csv-filename to store lookahead-analysis- + results +``` + +!!! Note "" + The above Output was reduced to options `lookahead-analysis` adds on top of regular backtesting commands. + +### Summary + +Checks a given strategy for look ahead bias via lookahead-analysis +Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting +and producing false hopes for the one backtesting. + +### Introduction + +Many strategies - without the programmer knowing - have fallen prey to look ahead bias. + +Any backtest will populate the full dataframe including all time stamps at the beginning. +If the programmer is not careful or oblivious how things work internally +(which sometimes can be really hard to find out) then it will just look into the future making the strategy amazing +but not realistic. + +This command is made to try to verify the validity in the form of the aforementioned look ahead bias. + +### How does the command work? + +It will start with a backtest of all pairs to generate a baseline for indicators and entries/exits. +After the backtest ran, it will look if the `minimum-trade-amount` is met +and if not cancel the lookahead-analysis for this strategy. + +After setting the baseline it will then do additional runs for every entry and exit separately. +When a verification-backtest is done, it will compare the indicators as the signal (either entry or exit) and report the bias. +After all signals have been verified or falsified a result-table will be generated for the user to see. + +### Caveats + +- `lookahead-analysis` can only verify / falsify the trades it calculated and verified. +If the strategy has many different signals / signal types, it's up to you to select appropriate parameters to ensure that all signals have triggered at least once. Not triggered signals will not have been verified. +This could lead to a false-negative (the strategy will then be reported as non-biased). +- `lookahead-analysis` has access to everything that backtesting has too. +Please don't provoke any configs like enabling position stacking. +If you decide to do so, then make doubly sure that you won't ever run out of `max_open_trades` amount and neither leftover money in your wallet. From 37fa186c556a783c2f2719040adce648a81acfd6 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 20 Sep 2023 22:43:01 +0900 Subject: [PATCH 23/52] remove 1 column --- docs/recursive-analysis.md | 21 ------------------- .../optimize/recursive_analysis_helpers.py | 4 ++-- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 3a8b3017c..86b358830 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -16,8 +16,6 @@ head over to the [Data Downloading](data-download.md) section of the documentati This command is built upon backtesting since it internally chains backtests to prepare different lenghts of data and calculate indicators based of each of the prepared data. This is done by not looking at the strategy itself - but at the value of the indicators it returned. After multiple backtests are done to calculate the indicators of different startup candles value, the values of last rows are compared to see hoe much differences are they compared to the base backtest. -You can use commands of [Backtesting](backtesting.md). - - `--cache` is forced to "none". - Since we are only looking at indicators' value, using more than one pair is redundant. It is recommended to set the pair used in the command using `-p` flag, preferably using pair with high price, such as BTC or ETH, to avoid having rounding issue that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis the first pair in the whitelist. - It's recommended to set a long timerange (at least consist of 5000 candles), so that the initial backtest that going to be used as benchmark have very small or no recursive issue at all. For example, for a 5m timeframe, timerange of 5000 candles would be equal to 18 days. @@ -36,29 +34,10 @@ usage: freqtrade recursive-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--timerange TIMERANGE] [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}] [-p PAIRS [PAIRS ...]] - [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] - [--export {none,trades,signals}] - [--export-filename PATH] - [--breakdown {day,week,month} [{day,week,month} ...]] - [--cache {none,day,week,month}] [--freqai-backtest-live-models] - [--minimum-trade-amount INT] - [--targeted-trade-amount INT] - [--lookahead-analysis-exportfilename LOOKAHEAD_ANALYSIS_EXPORTFILENAME] -options: - --minimum-trade-amount INT - Minimum trade amount for lookahead-analysis - --targeted-trade-amount INT - Targeted trade amount for lookahead analysis - --lookahead-analysis-exportfilename LOOKAHEAD_ANALYSIS_EXPORTFILENAME - Use this csv-filename to store lookahead-analysis- - results ``` -!!! Note "" - The above Output was reduced to options `lookahead-analysis` adds on top of regular backtesting commands. - ### Summary Checks a given strategy for look ahead bias via lookahead-analysis diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py index 8bae36dd2..167ea5386 100644 --- a/freqtrade/optimize/recursive_analysis_helpers.py +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -19,7 +19,7 @@ class RecursiveAnalysisSubFunctions: config: Dict[str, Any], recursive_instances: List[RecursiveAnalysis]): startups = recursive_instances[0]._startup_candle - headers = ['strategy', 'indicators'] + headers = ['indicators'] for candle in startups: headers.append(candle) @@ -27,7 +27,7 @@ class RecursiveAnalysisSubFunctions: for inst in recursive_instances: if len(inst.dict_recursive) > 0: for indicator, values in inst.dict_recursive.items(): - temp_data = [inst.strategy_obj['name'], indicator] + temp_data = [indicator] for candle in startups: temp_data.append(values.get(int(candle), '-')) data.append(temp_data) From e77b9de89e799b1598ca7ab0548d91dc967cb5fe Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 21 Sep 2023 10:51:12 +0900 Subject: [PATCH 24/52] fix docs --- docs/recursive-analysis.md | 38 +++++++++----------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 86b358830..10c1fe077 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -17,10 +17,10 @@ This command is built upon backtesting since it internally chains backtests to p This is done by not looking at the strategy itself - but at the value of the indicators it returned. After multiple backtests are done to calculate the indicators of different startup candles value, the values of last rows are compared to see hoe much differences are they compared to the base backtest. - `--cache` is forced to "none". -- Since we are only looking at indicators' value, using more than one pair is redundant. It is recommended to set the pair used in the command using `-p` flag, preferably using pair with high price, such as BTC or ETH, to avoid having rounding issue that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis the first pair in the whitelist. +- Since we are only looking at indicators' value, using more than one pair is redundant. It is recommended to set the pair used in the command using `-p` flag, preferably using pair with high price, such as BTC or ETH, to avoid having rounding issue that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis is the first pair in the whitelist. - It's recommended to set a long timerange (at least consist of 5000 candles), so that the initial backtest that going to be used as benchmark have very small or no recursive issue at all. For example, for a 5m timeframe, timerange of 5000 candles would be equal to 18 days. -Beside recursive formula check, this command also going to do a simple lookahead bias check on the indicators' value only. It won't replace [Lookahead-analysis](lookahead-analysis.md), since this check won't check the difference in trades' entries and exits. It will only check whether there is any difference in indicators' value if the end of the data are moved. +Beside recursive formula check, this command also going to do a simple lookahead bias check on the indicators' value only. It won't replace [Lookahead-analysis](lookahead-analysis.md), since this check won't check the difference in trades' entries and exits, which is the important effect of lookahead bias. It will only check whether there is any lookahead bias in indicators if the end of the data are moved. ## Recursive-analysis command reference @@ -40,36 +40,16 @@ usage: freqtrade recursive-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] ### Summary -Checks a given strategy for look ahead bias via lookahead-analysis -Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting -and producing false hopes for the one backtesting. - -### Introduction - -Many strategies - without the programmer knowing - have fallen prey to look ahead bias. - -Any backtest will populate the full dataframe including all time stamps at the beginning. -If the programmer is not careful or oblivious how things work internally -(which sometimes can be really hard to find out) then it will just look into the future making the strategy amazing -but not realistic. - -This command is made to try to verify the validity in the form of the aforementioned look ahead bias. +Checks a given strategy for recursive formula issue via recursive-analysis. +Recursive formula issue means that the indicator's calculation don't have enough data for its calculation to produce correct value. ### How does the command work? -It will start with a backtest of all pairs to generate a baseline for indicators and entries/exits. -After the backtest ran, it will look if the `minimum-trade-amount` is met -and if not cancel the lookahead-analysis for this strategy. - -After setting the baseline it will then do additional runs for every entry and exit separately. -When a verification-backtest is done, it will compare the indicators as the signal (either entry or exit) and report the bias. -After all signals have been verified or falsified a result-table will be generated for the user to see. +It will start with a backtest using the supplied timerange to generate a baseline for indicators' value. +After setting the baseline it will then do additional runs for each different startup candles. +When the additional runs are done, it will compare the indicators at the last rows and report the differences in a table. ### Caveats -- `lookahead-analysis` can only verify / falsify the trades it calculated and verified. -If the strategy has many different signals / signal types, it's up to you to select appropriate parameters to ensure that all signals have triggered at least once. Not triggered signals will not have been verified. -This could lead to a false-negative (the strategy will then be reported as non-biased). -- `lookahead-analysis` has access to everything that backtesting has too. -Please don't provoke any configs like enabling position stacking. -If you decide to do so, then make doubly sure that you won't ever run out of `max_open_trades` amount and neither leftover money in your wallet. +- `recursive-analysis` will only calculate and compare the indicators' value at the last row. If there are any differences, the table will only tell you the percentage differences. Whether it has any real impact on your entries and exits isn't checked. +- The ideal scenario is to have your indicators have no difference at all despite the startup candle being varied. But in reality, some of publicly-available formulas are using recursive formula. So the goal isn't to have zero differences, but to have the differences low enough to make sure they won't have any real impact on trading decisions. From 32b0098ec11985b352cfb48a38d1e34522e65806 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 21 Sep 2023 11:08:47 +0900 Subject: [PATCH 25/52] fix example in the docs, increasing startup to 400 on ema100 --- docs/recursive-analysis.md | 2 +- docs/strategy-customization.md | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 10c1fe077..341493f7e 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -1,4 +1,4 @@ -# Lookahead analysis +# Recursive analysis This page explains how to validate your strategy in terms of recursive formula issue. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 005127715..e23c3cc41 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -168,10 +168,12 @@ Most indicators have an instable startup period, in which they are either not av To account for this, the strategy can be assigned the `startup_candle_count` attribute. This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. In the case where a user includes higher timeframes with informative pairs, the `startup_candle_count` does not necessarily change. The value is the maximum period (in candles) that any of the informatives timeframes need to compute stable indicators. -In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles. +You can use [recursive-analysis](recursive-analysis.md) to check and find the correct `startup_candle_count` to be used. + +In this example strategy, this should be set to 400 (`startup_candle_count = 400`), since the minimum needed history for ema100 calculation to make sure the value is correct is 400 candles. ``` python - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=400) ``` By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt. @@ -193,11 +195,11 @@ Let's try to backtest 1 month (January 2019) of 5m candles using an example stra freqtrade backtesting --timerange 20190101-20190201 --timeframe 5m ``` -Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2018-12-31 15:30:00. +Assuming `startup_candle_count` is set to 400, backtesting knows it needs 400 candles to generate valid buy signals. It will load data from `20190101 - (400 * 5m)` - which is ~2018-12-30 11:40:00. If this data is available, indicators will be calculated with this extended timerange. The instable startup period (up to 2019-01-01 00:00:00) will then be removed before starting backtesting. !!! Note - If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00. + If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-02 09:20:00. ### Entry signal rules From d465fcffd5a70d46b70fb36937619776f9bbd46c Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 21 Sep 2023 11:13:06 +0900 Subject: [PATCH 26/52] change startup cande in sample strat --- freqtrade/templates/sample_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index fd81570fe..65a6e440e 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -77,7 +77,7 @@ class SampleStrategy(IStrategy): exit_short_rsi = IntParameter(low=1, high=50, default=30, space='buy', optimize=True, load=True) # Number of candles the strategy requires before producing valid signals - startup_candle_count: int = 30 + startup_candle_count: int = 170 # Optional order type mapping. order_types = { From 28e43a486783855933da7121fbc6f63e4be3b3ce Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 21 Sep 2023 14:00:17 +0900 Subject: [PATCH 27/52] initial test --- tests/optimize/test_recursive_analysis.py | 370 ++++++++++++++++++ .../strategy_test_v3_recursive_issue.py | 43 ++ 2 files changed, 413 insertions(+) create mode 100644 tests/optimize/test_recursive_analysis.py create mode 100644 tests/strategy/strats/strategy_test_v3_recursive_issue.py diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py new file mode 100644 index 000000000..021d3eb72 --- /dev/null +++ b/tests/optimize/test_recursive_analysis.py @@ -0,0 +1,370 @@ +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument +from copy import deepcopy +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock + +import pytest + +from freqtrade.commands.optimize_commands import start_recursive_analysis +from freqtrade.data.history import get_timerange +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.recursive_analysis import Analysis, RecursiveAnalysis +from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions +from tests.conftest import EXMS, get_args, log_has_re, patch_exchange + + +@pytest.fixture +def recursive_conf(default_conf_usdt): + default_conf_usdt['timerange'] = '20220101-20220501' + + default_conf_usdt['strategy_path'] = str( + Path(__file__).parent.parent / "strategy/strats") + default_conf_usdt['strategy'] = 'strategy_test_v3_recursive_issue' + default_conf_usdt['pairs'] = ['UNITTEST/USDT'] + return default_conf_usdt + + +def test_start_recursive_analysis(mocker): + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', + initialize_single_recursive_analysis=single_mock, + text_table_recursive_analysis_instances=text_table_mock, + ) + args = [ + "recursive-analysis", + "--strategy", + "strategy_test_v3_recursive_issue", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats"), + "--pairs", + "UNITTEST/BTC", + "--timerange", + "20220101-20220201" + ] + pargs = get_args(args) + pargs['config'] = None + + start_recursive_analysis(pargs) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 + + single_mock.reset_mock() + + # Test invalid config + args = [ + "lookahead-analysis", + "--strategy", + "strategy_test_v3_with_lookahead_bias", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), + "--targeted-trade-amount", + "10", + "--minimum-trade-amount", + "20", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): + start_lookahead_analysis(pargs) + + # Missing timerange + args = [ + "lookahead-analysis", + "--strategy", + "strategy_test_v3_with_lookahead_bias", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), + "--pairs", + "UNITTEST/BTC", + "--max-open-trades", + "1", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"Please set a timerange\..*"): + start_lookahead_analysis(pargs) + + +def test_lookahead_helper_invalid_config(lookahead_conf) -> None: + conf = deepcopy(lookahead_conf) + conf['targeted_trade_amount'] = 10 + conf['minimum_trade_amount'] = 40 + with pytest.raises(OperationalException, + match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): + LookaheadAnalysisSubFunctions.start(conf) + + +def test_lookahead_helper_no_strategy_defined(lookahead_conf): + conf = deepcopy(lookahead_conf) + conf['pairs'] = ['UNITTEST/USDT'] + del conf['strategy'] + with pytest.raises(OperationalException, + match=r"No Strategy specified"): + LookaheadAnalysisSubFunctions.start(conf) + + +def test_lookahead_helper_start(lookahead_conf, mocker) -> None: + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.lookahead_analysis_helpers.LookaheadAnalysisSubFunctions', + initialize_single_lookahead_analysis=single_mock, + text_table_lookahead_analysis_instances=text_table_mock, + ) + LookaheadAnalysisSubFunctions.start(lookahead_conf) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 + + single_mock.reset_mock() + text_table_mock.reset_mock() + + +def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf): + analysis = Analysis() + analysis.has_bias = True + analysis.total_signals = 5 + analysis.false_entry_signals = 4 + analysis.false_exit_signals = 3 + + strategy_obj = { + 'name': "strategy_test_v3_with_lookahead_bias", + 'location': Path(lookahead_conf['strategy_path'], f"{lookahead_conf['strategy']}.py") + } + + instance = LookaheadAnalysis(lookahead_conf, strategy_obj) + instance.current_analysis = analysis + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + + # check row contents for a try that has too few signals + assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' + assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' + assert data[0][2].__contains__('too few trades') + assert len(data[0]) == 3 + + # now check for an error which occured after enough trades + analysis.total_signals = 12 + analysis.false_entry_signals = 11 + analysis.false_exit_signals = 10 + instance = LookaheadAnalysis(lookahead_conf, strategy_obj) + instance.current_analysis = analysis + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + assert data[0][2].__contains__("error") + + # edit it into not showing an error + instance.failed_bias_check = False + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' + assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' + assert data[0][2] # True + assert data[0][3] == 12 + assert data[0][4] == 11 + assert data[0][5] == 10 + assert data[0][6] == '' + + analysis.false_indicators.append('falseIndicator1') + analysis.false_indicators.append('falseIndicator2') + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + + assert data[0][6] == 'falseIndicator1, falseIndicator2' + + # check amount of returning rows + assert len(data) == 1 + + # check amount of multiple rows + table, headers, data = (LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + lookahead_conf, [instance, instance, instance])) + assert len(data) == 3 + + +def test_lookahead_helper_export_to_csv(lookahead_conf): + import pandas as pd + lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" + + # just to be sure the test won't fail: remove file if exists for some reason + # (repeat this at the end once again to clean up) + if Path(lookahead_conf['lookahead_analysis_exportfilename']).exists(): + Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() + + # before we can start we have to delete the + + # 1st check: create a new file and verify its contents + analysis1 = Analysis() + analysis1.has_bias = True + analysis1.total_signals = 12 + analysis1.false_entry_signals = 11 + analysis1.false_exit_signals = 10 + analysis1.false_indicators.append('falseIndicator1') + analysis1.false_indicators.append('falseIndicator2') + lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" + + strategy_obj1 = { + 'name': "strat1", + 'location': Path("file1.py"), + } + + instance1 = LookaheadAnalysis(lookahead_conf, strategy_obj1) + instance1.failed_bias_check = False + instance1.current_analysis = analysis1 + + LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance1]) + saved_data1 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) + + expected_values1 = [ + [ + 'file1.py', 'strat1', True, + 12, 11, 10, + "falseIndicator1,falseIndicator2" + ], + ] + expected_columns = ['filename', 'strategy', 'has_bias', + 'total_signals', 'biased_entry_signals', 'biased_exit_signals', + 'biased_indicators'] + expected_data1 = pd.DataFrame(expected_values1, columns=expected_columns) + + assert Path(lookahead_conf['lookahead_analysis_exportfilename']).exists() + assert expected_data1.equals(saved_data1) + + # 2nd check: update the same strategy (which internally changed or is being retested) + expected_values2 = [ + [ + 'file1.py', 'strat1', False, + 22, 21, 20, + "falseIndicator3,falseIndicator4" + ], + ] + expected_data2 = pd.DataFrame(expected_values2, columns=expected_columns) + + analysis2 = Analysis() + analysis2.has_bias = False + analysis2.total_signals = 22 + analysis2.false_entry_signals = 21 + analysis2.false_exit_signals = 20 + analysis2.false_indicators.append('falseIndicator3') + analysis2.false_indicators.append('falseIndicator4') + + strategy_obj2 = { + 'name': "strat1", + 'location': Path("file1.py"), + } + + instance2 = LookaheadAnalysis(lookahead_conf, strategy_obj2) + instance2.failed_bias_check = False + instance2.current_analysis = analysis2 + + LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance2]) + saved_data2 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) + + assert expected_data2.equals(saved_data2) + + # 3rd check: now we add a new row to an already existing file + expected_values3 = [ + [ + 'file1.py', 'strat1', False, + 22, 21, 20, + "falseIndicator3,falseIndicator4" + ], + [ + 'file3.py', 'strat3', True, + 32, 31, 30, "falseIndicator5,falseIndicator6" + ], + ] + + expected_data3 = pd.DataFrame(expected_values3, columns=expected_columns) + + analysis3 = Analysis() + analysis3.has_bias = True + analysis3.total_signals = 32 + analysis3.false_entry_signals = 31 + analysis3.false_exit_signals = 30 + analysis3.false_indicators.append('falseIndicator5') + analysis3.false_indicators.append('falseIndicator6') + lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" + + strategy_obj3 = { + 'name': "strat3", + 'location': Path("file3.py"), + } + + instance3 = LookaheadAnalysis(lookahead_conf, strategy_obj3) + instance3.failed_bias_check = False + instance3.current_analysis = analysis3 + + LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance3]) + saved_data3 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) + assert expected_data3.equals(saved_data3) + + # remove csv file after the test is done + if Path(lookahead_conf['lookahead_analysis_exportfilename']).exists(): + Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() + + +def test_initialize_single_lookahead_analysis(lookahead_conf, mocker, caplog): + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + mocker.patch(f'{EXMS}.get_fee', return_value=0.0) + mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) + mocker.patch(f'{EXMS}.get_max_pair_stake_amount', return_value=float('inf')) + patch_exchange(mocker) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + lookahead_conf['pairs'] = ['UNITTEST/USDT'] + + lookahead_conf['timeframe'] = '5m' + lookahead_conf['timerange'] = '20180119-20180122' + start_mock = mocker.patch('freqtrade.optimize.lookahead_analysis.LookaheadAnalysis.start') + strategy_obj = { + 'name': "strategy_test_v3_with_lookahead_bias", + 'location': Path(lookahead_conf['strategy_path'], f"{lookahead_conf['strategy']}.py") + } + + instance = LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( + lookahead_conf, strategy_obj) + assert log_has_re(r"Bias test of .* started\.", caplog) + assert start_mock.call_count == 1 + + assert instance.strategy_obj['name'] == "strategy_test_v3_with_lookahead_bias" + + +@pytest.mark.parametrize('scenario', [ + 'no_bias', 'bias1' +]) +def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + patch_exchange(mocker) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + lookahead_conf['pairs'] = ['UNITTEST/USDT'] + + lookahead_conf['timeframe'] = '5m' + lookahead_conf['timerange'] = '20180119-20180122' + lookahead_conf['startup_candle'] = [100] + + # Patch scenario Parameter to allow for easy selection + mocker.patch('freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file', + return_value={ + 'params': { + "buy": { + "scenario": scenario + } + } + }) + + strategy_obj = {'name': "strategy_test_v3_recursive_issue"} + instance = RecursiveAnalysis(lookahead_conf, strategy_obj) + instance.start() + # Assert init correct + assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog) + + # check non-biased strategy + if scenario == "no_bias": + assert not instance.current_analysis.has_bias + # check biased strategy + elif scenario == "bias1": + assert instance.current_analysis.has_bias diff --git a/tests/strategy/strats/strategy_test_v3_recursive_issue.py b/tests/strategy/strats/strategy_test_v3_recursive_issue.py new file mode 100644 index 000000000..c4ddc6a1b --- /dev/null +++ b/tests/strategy/strats/strategy_test_v3_recursive_issue.py @@ -0,0 +1,43 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement +from pandas import DataFrame +from technical.indicators import ichimoku + +from freqtrade.strategy import IStrategy +from freqtrade.strategy.parameters import CategoricalParameter + +import talib.abstract as ta + +class strategy_test_v3_recursive_issue(IStrategy): + INTERFACE_VERSION = 3 + + # Minimal ROI designed for the strategy + minimal_roi = { + "0": 0.04 + } + + # Optimal stoploss designed for the strategy + stoploss = -0.10 + + # Optimal timeframe for the strategy + timeframe = '5m' + scenario = CategoricalParameter(['no_bias', 'bias1'], default='bias1', space="buy") + + # Number of candles the strategy requires before producing valid signals + startup_candle_count: int = 100 + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # bias is introduced here + if self.scenario.value == 'no_bias': + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + else: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=50) + + return dataframe + + def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + return dataframe + + def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + return dataframe From 08b94a20778d808af03fd107108b86155437fd4f Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 14:21:54 +0900 Subject: [PATCH 28/52] 1 test --- tests/optimize/test_recursive_analysis.py | 527 +++++++++++----------- 1 file changed, 264 insertions(+), 263 deletions(-) diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 021d3eb72..458c07829 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -24,312 +24,312 @@ def recursive_conf(default_conf_usdt): return default_conf_usdt -def test_start_recursive_analysis(mocker): - single_mock = MagicMock() - text_table_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', - initialize_single_recursive_analysis=single_mock, - text_table_recursive_analysis_instances=text_table_mock, - ) - args = [ - "recursive-analysis", - "--strategy", - "strategy_test_v3_recursive_issue", - "--strategy-path", - str(Path(__file__).parent.parent / "strategy/strats"), - "--pairs", - "UNITTEST/BTC", - "--timerange", - "20220101-20220201" - ] - pargs = get_args(args) - pargs['config'] = None +# def test_start_recursive_analysis(mocker): +# single_mock = MagicMock() +# text_table_mock = MagicMock() +# mocker.patch.multiple( +# 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', +# initialize_single_recursive_analysis=single_mock, +# text_table_recursive_analysis_instances=text_table_mock, +# ) +# args = [ +# "recursive-analysis", +# "--strategy", +# "strategy_test_v3_recursive_issue", +# "--strategy-path", +# str(Path(__file__).parent.parent / "strategy/strats"), +# "--pairs", +# "UNITTEST/BTC", +# "--timerange", +# "20220101-20220201" +# ] +# pargs = get_args(args) +# pargs['config'] = None - start_recursive_analysis(pargs) - assert single_mock.call_count == 1 - assert text_table_mock.call_count == 1 +# start_recursive_analysis(pargs) +# assert single_mock.call_count == 1 +# assert text_table_mock.call_count == 1 - single_mock.reset_mock() +# single_mock.reset_mock() - # Test invalid config - args = [ - "lookahead-analysis", - "--strategy", - "strategy_test_v3_with_lookahead_bias", - "--strategy-path", - str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), - "--targeted-trade-amount", - "10", - "--minimum-trade-amount", - "20", - ] - pargs = get_args(args) - pargs['config'] = None - with pytest.raises(OperationalException, - match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): - start_lookahead_analysis(pargs) +# # Test invalid config +# args = [ +# "lookahead-analysis", +# "--strategy", +# "strategy_test_v3_with_lookahead_bias", +# "--strategy-path", +# str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), +# "--targeted-trade-amount", +# "10", +# "--minimum-trade-amount", +# "20", +# ] +# pargs = get_args(args) +# pargs['config'] = None +# with pytest.raises(OperationalException, +# match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): +# start_lookahead_analysis(pargs) - # Missing timerange - args = [ - "lookahead-analysis", - "--strategy", - "strategy_test_v3_with_lookahead_bias", - "--strategy-path", - str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), - "--pairs", - "UNITTEST/BTC", - "--max-open-trades", - "1", - ] - pargs = get_args(args) - pargs['config'] = None - with pytest.raises(OperationalException, - match=r"Please set a timerange\..*"): - start_lookahead_analysis(pargs) +# # Missing timerange +# args = [ +# "lookahead-analysis", +# "--strategy", +# "strategy_test_v3_with_lookahead_bias", +# "--strategy-path", +# str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), +# "--pairs", +# "UNITTEST/BTC", +# "--max-open-trades", +# "1", +# ] +# pargs = get_args(args) +# pargs['config'] = None +# with pytest.raises(OperationalException, +# match=r"Please set a timerange\..*"): +# start_lookahead_analysis(pargs) -def test_lookahead_helper_invalid_config(lookahead_conf) -> None: - conf = deepcopy(lookahead_conf) - conf['targeted_trade_amount'] = 10 - conf['minimum_trade_amount'] = 40 - with pytest.raises(OperationalException, - match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): - LookaheadAnalysisSubFunctions.start(conf) +# def test_lookahead_helper_invalid_config(recursive_conf) -> None: +# conf = deepcopy(recursive_conf) +# conf['targeted_trade_amount'] = 10 +# conf['minimum_trade_amount'] = 40 +# with pytest.raises(OperationalException, +# match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): +# RecursiveAnalysisSubFunctions.start(conf) -def test_lookahead_helper_no_strategy_defined(lookahead_conf): - conf = deepcopy(lookahead_conf) - conf['pairs'] = ['UNITTEST/USDT'] - del conf['strategy'] - with pytest.raises(OperationalException, - match=r"No Strategy specified"): - LookaheadAnalysisSubFunctions.start(conf) +# def test_lookahead_helper_no_strategy_defined(recursive_conf): +# conf = deepcopy(recursive_conf) +# conf['pairs'] = ['UNITTEST/USDT'] +# del conf['strategy'] +# with pytest.raises(OperationalException, +# match=r"No Strategy specified"): +# RecursiveAnalysisSubFunctions.start(conf) -def test_lookahead_helper_start(lookahead_conf, mocker) -> None: - single_mock = MagicMock() - text_table_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.optimize.lookahead_analysis_helpers.LookaheadAnalysisSubFunctions', - initialize_single_lookahead_analysis=single_mock, - text_table_lookahead_analysis_instances=text_table_mock, - ) - LookaheadAnalysisSubFunctions.start(lookahead_conf) - assert single_mock.call_count == 1 - assert text_table_mock.call_count == 1 +# def test_lookahead_helper_start(recursive_conf, mocker) -> None: +# single_mock = MagicMock() +# text_table_mock = MagicMock() +# mocker.patch.multiple( +# 'freqtrade.optimize.lookahead_analysis_helpers.RecursiveAnalysisSubFunctions', +# initialize_single_lookahead_analysis=single_mock, +# text_table_lookahead_analysis_instances=text_table_mock, +# ) +# RecursiveAnalysisSubFunctions.start(recursive_conf) +# assert single_mock.call_count == 1 +# assert text_table_mock.call_count == 1 - single_mock.reset_mock() - text_table_mock.reset_mock() +# single_mock.reset_mock() +# text_table_mock.reset_mock() -def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf): - analysis = Analysis() - analysis.has_bias = True - analysis.total_signals = 5 - analysis.false_entry_signals = 4 - analysis.false_exit_signals = 3 +# def test_lookahead_helper_text_table_lookahead_analysis_instances(recursive_conf): +# analysis = Analysis() +# analysis.has_bias = True +# analysis.total_signals = 5 +# analysis.false_entry_signals = 4 +# analysis.false_exit_signals = 3 - strategy_obj = { - 'name': "strategy_test_v3_with_lookahead_bias", - 'location': Path(lookahead_conf['strategy_path'], f"{lookahead_conf['strategy']}.py") - } +# strategy_obj = { +# 'name': "strategy_test_v3_with_lookahead_bias", +# 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") +# } - instance = LookaheadAnalysis(lookahead_conf, strategy_obj) - instance.current_analysis = analysis - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) +# instance = LookaheadAnalysis(recursive_conf, strategy_obj) +# instance.current_analysis = analysis +# table, headers, data = (RecursiveAnalysisSubFunctions. +# text_table_lookahead_analysis_instances(recursive_conf, [instance])) - # check row contents for a try that has too few signals - assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' - assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' - assert data[0][2].__contains__('too few trades') - assert len(data[0]) == 3 +# # check row contents for a try that has too few signals +# assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' +# assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' +# assert data[0][2].__contains__('too few trades') +# assert len(data[0]) == 3 - # now check for an error which occured after enough trades - analysis.total_signals = 12 - analysis.false_entry_signals = 11 - analysis.false_exit_signals = 10 - instance = LookaheadAnalysis(lookahead_conf, strategy_obj) - instance.current_analysis = analysis - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) - assert data[0][2].__contains__("error") +# # now check for an error which occured after enough trades +# analysis.total_signals = 12 +# analysis.false_entry_signals = 11 +# analysis.false_exit_signals = 10 +# instance = LookaheadAnalysis(recursive_conf, strategy_obj) +# instance.current_analysis = analysis +# table, headers, data = (RecursiveAnalysisSubFunctions. +# text_table_lookahead_analysis_instances(recursive_conf, [instance])) +# assert data[0][2].__contains__("error") - # edit it into not showing an error - instance.failed_bias_check = False - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) - assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' - assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' - assert data[0][2] # True - assert data[0][3] == 12 - assert data[0][4] == 11 - assert data[0][5] == 10 - assert data[0][6] == '' +# # edit it into not showing an error +# instance.failed_bias_check = False +# table, headers, data = (RecursiveAnalysisSubFunctions. +# text_table_lookahead_analysis_instances(recursive_conf, [instance])) +# assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' +# assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' +# assert data[0][2] # True +# assert data[0][3] == 12 +# assert data[0][4] == 11 +# assert data[0][5] == 10 +# assert data[0][6] == '' - analysis.false_indicators.append('falseIndicator1') - analysis.false_indicators.append('falseIndicator2') - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) +# analysis.false_indicators.append('falseIndicator1') +# analysis.false_indicators.append('falseIndicator2') +# table, headers, data = (RecursiveAnalysisSubFunctions. +# text_table_lookahead_analysis_instances(recursive_conf, [instance])) - assert data[0][6] == 'falseIndicator1, falseIndicator2' +# assert data[0][6] == 'falseIndicator1, falseIndicator2' - # check amount of returning rows - assert len(data) == 1 +# # check amount of returning rows +# assert len(data) == 1 - # check amount of multiple rows - table, headers, data = (LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( - lookahead_conf, [instance, instance, instance])) - assert len(data) == 3 +# # check amount of multiple rows +# table, headers, data = (RecursiveAnalysisSubFunctions.text_table_lookahead_analysis_instances( +# recursive_conf, [instance, instance, instance])) +# assert len(data) == 3 -def test_lookahead_helper_export_to_csv(lookahead_conf): - import pandas as pd - lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" +# def test_lookahead_helper_export_to_csv(recursive_conf): +# import pandas as pd +# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" - # just to be sure the test won't fail: remove file if exists for some reason - # (repeat this at the end once again to clean up) - if Path(lookahead_conf['lookahead_analysis_exportfilename']).exists(): - Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() +# # just to be sure the test won't fail: remove file if exists for some reason +# # (repeat this at the end once again to clean up) +# if Path(recursive_conf['lookahead_analysis_exportfilename']).exists(): +# Path(recursive_conf['lookahead_analysis_exportfilename']).unlink() - # before we can start we have to delete the +# # before we can start we have to delete the - # 1st check: create a new file and verify its contents - analysis1 = Analysis() - analysis1.has_bias = True - analysis1.total_signals = 12 - analysis1.false_entry_signals = 11 - analysis1.false_exit_signals = 10 - analysis1.false_indicators.append('falseIndicator1') - analysis1.false_indicators.append('falseIndicator2') - lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" +# # 1st check: create a new file and verify its contents +# analysis1 = Analysis() +# analysis1.has_bias = True +# analysis1.total_signals = 12 +# analysis1.false_entry_signals = 11 +# analysis1.false_exit_signals = 10 +# analysis1.false_indicators.append('falseIndicator1') +# analysis1.false_indicators.append('falseIndicator2') +# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" - strategy_obj1 = { - 'name': "strat1", - 'location': Path("file1.py"), - } +# strategy_obj1 = { +# 'name': "strat1", +# 'location': Path("file1.py"), +# } - instance1 = LookaheadAnalysis(lookahead_conf, strategy_obj1) - instance1.failed_bias_check = False - instance1.current_analysis = analysis1 +# instance1 = LookaheadAnalysis(recursive_conf, strategy_obj1) +# instance1.failed_bias_check = False +# instance1.current_analysis = analysis1 - LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance1]) - saved_data1 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) +# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance1]) +# saved_data1 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) - expected_values1 = [ - [ - 'file1.py', 'strat1', True, - 12, 11, 10, - "falseIndicator1,falseIndicator2" - ], - ] - expected_columns = ['filename', 'strategy', 'has_bias', - 'total_signals', 'biased_entry_signals', 'biased_exit_signals', - 'biased_indicators'] - expected_data1 = pd.DataFrame(expected_values1, columns=expected_columns) +# expected_values1 = [ +# [ +# 'file1.py', 'strat1', True, +# 12, 11, 10, +# "falseIndicator1,falseIndicator2" +# ], +# ] +# expected_columns = ['filename', 'strategy', 'has_bias', +# 'total_signals', 'biased_entry_signals', 'biased_exit_signals', +# 'biased_indicators'] +# expected_data1 = pd.DataFrame(expected_values1, columns=expected_columns) - assert Path(lookahead_conf['lookahead_analysis_exportfilename']).exists() - assert expected_data1.equals(saved_data1) +# assert Path(recursive_conf['lookahead_analysis_exportfilename']).exists() +# assert expected_data1.equals(saved_data1) - # 2nd check: update the same strategy (which internally changed or is being retested) - expected_values2 = [ - [ - 'file1.py', 'strat1', False, - 22, 21, 20, - "falseIndicator3,falseIndicator4" - ], - ] - expected_data2 = pd.DataFrame(expected_values2, columns=expected_columns) +# # 2nd check: update the same strategy (which internally changed or is being retested) +# expected_values2 = [ +# [ +# 'file1.py', 'strat1', False, +# 22, 21, 20, +# "falseIndicator3,falseIndicator4" +# ], +# ] +# expected_data2 = pd.DataFrame(expected_values2, columns=expected_columns) - analysis2 = Analysis() - analysis2.has_bias = False - analysis2.total_signals = 22 - analysis2.false_entry_signals = 21 - analysis2.false_exit_signals = 20 - analysis2.false_indicators.append('falseIndicator3') - analysis2.false_indicators.append('falseIndicator4') +# analysis2 = Analysis() +# analysis2.has_bias = False +# analysis2.total_signals = 22 +# analysis2.false_entry_signals = 21 +# analysis2.false_exit_signals = 20 +# analysis2.false_indicators.append('falseIndicator3') +# analysis2.false_indicators.append('falseIndicator4') - strategy_obj2 = { - 'name': "strat1", - 'location': Path("file1.py"), - } +# strategy_obj2 = { +# 'name': "strat1", +# 'location': Path("file1.py"), +# } - instance2 = LookaheadAnalysis(lookahead_conf, strategy_obj2) - instance2.failed_bias_check = False - instance2.current_analysis = analysis2 +# instance2 = LookaheadAnalysis(recursive_conf, strategy_obj2) +# instance2.failed_bias_check = False +# instance2.current_analysis = analysis2 - LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance2]) - saved_data2 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) +# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance2]) +# saved_data2 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) - assert expected_data2.equals(saved_data2) +# assert expected_data2.equals(saved_data2) - # 3rd check: now we add a new row to an already existing file - expected_values3 = [ - [ - 'file1.py', 'strat1', False, - 22, 21, 20, - "falseIndicator3,falseIndicator4" - ], - [ - 'file3.py', 'strat3', True, - 32, 31, 30, "falseIndicator5,falseIndicator6" - ], - ] +# # 3rd check: now we add a new row to an already existing file +# expected_values3 = [ +# [ +# 'file1.py', 'strat1', False, +# 22, 21, 20, +# "falseIndicator3,falseIndicator4" +# ], +# [ +# 'file3.py', 'strat3', True, +# 32, 31, 30, "falseIndicator5,falseIndicator6" +# ], +# ] - expected_data3 = pd.DataFrame(expected_values3, columns=expected_columns) +# expected_data3 = pd.DataFrame(expected_values3, columns=expected_columns) - analysis3 = Analysis() - analysis3.has_bias = True - analysis3.total_signals = 32 - analysis3.false_entry_signals = 31 - analysis3.false_exit_signals = 30 - analysis3.false_indicators.append('falseIndicator5') - analysis3.false_indicators.append('falseIndicator6') - lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" +# analysis3 = Analysis() +# analysis3.has_bias = True +# analysis3.total_signals = 32 +# analysis3.false_entry_signals = 31 +# analysis3.false_exit_signals = 30 +# analysis3.false_indicators.append('falseIndicator5') +# analysis3.false_indicators.append('falseIndicator6') +# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" - strategy_obj3 = { - 'name': "strat3", - 'location': Path("file3.py"), - } +# strategy_obj3 = { +# 'name': "strat3", +# 'location': Path("file3.py"), +# } - instance3 = LookaheadAnalysis(lookahead_conf, strategy_obj3) - instance3.failed_bias_check = False - instance3.current_analysis = analysis3 +# instance3 = LookaheadAnalysis(recursive_conf, strategy_obj3) +# instance3.failed_bias_check = False +# instance3.current_analysis = analysis3 - LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance3]) - saved_data3 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) - assert expected_data3.equals(saved_data3) +# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance3]) +# saved_data3 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) +# assert expected_data3.equals(saved_data3) - # remove csv file after the test is done - if Path(lookahead_conf['lookahead_analysis_exportfilename']).exists(): - Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() +# # remove csv file after the test is done +# if Path(recursive_conf['lookahead_analysis_exportfilename']).exists(): +# Path(recursive_conf['lookahead_analysis_exportfilename']).unlink() -def test_initialize_single_lookahead_analysis(lookahead_conf, mocker, caplog): - mocker.patch('freqtrade.data.history.get_timerange', get_timerange) - mocker.patch(f'{EXMS}.get_fee', return_value=0.0) - mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) - mocker.patch(f'{EXMS}.get_max_pair_stake_amount', return_value=float('inf')) - patch_exchange(mocker) - mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', - PropertyMock(return_value=['UNITTEST/BTC'])) - lookahead_conf['pairs'] = ['UNITTEST/USDT'] +# def test_initialize_single_lookahead_analysis(recursive_conf, mocker, caplog): +# mocker.patch('freqtrade.data.history.get_timerange', get_timerange) +# mocker.patch(f'{EXMS}.get_fee', return_value=0.0) +# mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) +# mocker.patch(f'{EXMS}.get_max_pair_stake_amount', return_value=float('inf')) +# patch_exchange(mocker) +# mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', +# PropertyMock(return_value=['UNITTEST/BTC'])) +# recursive_conf['pairs'] = ['UNITTEST/USDT'] - lookahead_conf['timeframe'] = '5m' - lookahead_conf['timerange'] = '20180119-20180122' - start_mock = mocker.patch('freqtrade.optimize.lookahead_analysis.LookaheadAnalysis.start') - strategy_obj = { - 'name': "strategy_test_v3_with_lookahead_bias", - 'location': Path(lookahead_conf['strategy_path'], f"{lookahead_conf['strategy']}.py") - } +# recursive_conf['timeframe'] = '5m' +# recursive_conf['timerange'] = '20180119-20180122' +# start_mock = mocker.patch('freqtrade.optimize.lookahead_analysis.LookaheadAnalysis.start') +# strategy_obj = { +# 'name': "strategy_test_v3_with_lookahead_bias", +# 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") +# } - instance = LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( - lookahead_conf, strategy_obj) - assert log_has_re(r"Bias test of .* started\.", caplog) - assert start_mock.call_count == 1 +# instance = RecursiveAnalysisSubFunctions.initialize_single_lookahead_analysis( +# recursive_conf, strategy_obj) +# assert log_has_re(r"Bias test of .* started\.", caplog) +# assert start_mock.call_count == 1 - assert instance.strategy_obj['name'] == "strategy_test_v3_with_lookahead_bias" +# assert instance.strategy_obj['name'] == "strategy_test_v3_with_lookahead_bias" @pytest.mark.parametrize('scenario', [ @@ -340,11 +340,11 @@ def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: patch_exchange(mocker) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) - lookahead_conf['pairs'] = ['UNITTEST/USDT'] + recursive_conf['pairs'] = ['UNITTEST/USDT'] - lookahead_conf['timeframe'] = '5m' - lookahead_conf['timerange'] = '20180119-20180122' - lookahead_conf['startup_candle'] = [100] + recursive_conf['timeframe'] = '5m' + recursive_conf['timerange'] = '20180119-20180122' + recursive_conf['startup_candle'] = [100] # Patch scenario Parameter to allow for easy selection mocker.patch('freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file', @@ -357,14 +357,15 @@ def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: }) strategy_obj = {'name': "strategy_test_v3_recursive_issue"} - instance = RecursiveAnalysis(lookahead_conf, strategy_obj) + instance = RecursiveAnalysis(recursive_conf, strategy_obj) instance.start() # Assert init correct assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog) + diff_pct = float(instance.dict_recursive['rsi'][100].replace("%", "")) # check non-biased strategy if scenario == "no_bias": - assert not instance.current_analysis.has_bias + assert diff_pct < 0.01 # check biased strategy elif scenario == "bias1": - assert instance.current_analysis.has_bias + assert diff_pct >= 0.01 From b9e9f82503a9baf5b0e290598b2f3bae0b8b2d46 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 16:45:43 +0900 Subject: [PATCH 29/52] first test done --- freqtrade/optimize/recursive_analysis.py | 3 +++ tests/optimize/test_recursive_analysis.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index 599fc4dda..45c2a457c 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -129,6 +129,7 @@ class RecursiveAnalysis(BaseAnalysis): varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) def fill_partial_varholder(self, start_date, startup_candle): + logger.info(f"Calculating indicators using startup candle of {startup_candle}.") partial_varHolder = VarHolder() partial_varHolder.from_dt = start_date @@ -142,6 +143,8 @@ class RecursiveAnalysis(BaseAnalysis): self.partial_varHolder_array.append(partial_varHolder) def fill_partial_varholder_lookahead(self, end_date): + logger.info("Calculating indicators to test lookahead on indicators.") + partial_varHolder = VarHolder() partial_varHolder.from_dt = self.full_varHolder.from_dt diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 458c07829..4d5ddc63a 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -8,7 +8,7 @@ import pytest from freqtrade.commands.optimize_commands import start_recursive_analysis from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException -from freqtrade.optimize.recursive_analysis import Analysis, RecursiveAnalysis +from freqtrade.optimize.recursive_analysis import RecursiveAnalysis from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions from tests.conftest import EXMS, get_args, log_has_re, patch_exchange @@ -340,7 +340,7 @@ def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: patch_exchange(mocker) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) - recursive_conf['pairs'] = ['UNITTEST/USDT'] + recursive_conf['pairs'] = ['UNITTEST/BTC'] recursive_conf['timeframe'] = '5m' recursive_conf['timerange'] = '20180119-20180122' @@ -362,7 +362,7 @@ def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: # Assert init correct assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog) - diff_pct = float(instance.dict_recursive['rsi'][100].replace("%", "")) + diff_pct = abs(float(instance.dict_recursive['rsi'][100].replace("%", ""))) # check non-biased strategy if scenario == "no_bias": assert diff_pct < 0.01 From 89d47ffd8fdca2372a231f1399f2e67065b72463 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 17:47:51 +0900 Subject: [PATCH 30/52] 4 more tests --- .../optimize/recursive_analysis_helpers.py | 3 +- tests/optimize/test_recursive_analysis.py | 286 ++++-------------- 2 files changed, 66 insertions(+), 223 deletions(-) diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py index 167ea5386..b33a24cf4 100644 --- a/freqtrade/optimize/recursive_analysis_helpers.py +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -16,7 +16,6 @@ class RecursiveAnalysisSubFunctions: @staticmethod def text_table_recursive_analysis_instances( - config: Dict[str, Any], recursive_instances: List[RecursiveAnalysis]): startups = recursive_instances[0]._startup_candle headers = ['indicators'] @@ -99,7 +98,7 @@ class RecursiveAnalysisSubFunctions: # report the results if RecursiveAnalysis_instances: RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances( - config, RecursiveAnalysis_instances) + RecursiveAnalysis_instances) else: logger.error("There were no strategies specified neither through " "--strategy nor through " diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 4d5ddc63a..d06cbdb90 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -21,6 +21,7 @@ def recursive_conf(default_conf_usdt): Path(__file__).parent.parent / "strategy/strats") default_conf_usdt['strategy'] = 'strategy_test_v3_recursive_issue' default_conf_usdt['pairs'] = ['UNITTEST/USDT'] + default_conf_usdt['startup_candle'] = [100] return default_conf_usdt @@ -54,11 +55,11 @@ def recursive_conf(default_conf_usdt): # # Test invalid config # args = [ -# "lookahead-analysis", +# "recursive-analysis", # "--strategy", -# "strategy_test_v3_with_lookahead_bias", +# "strategy_test_v3_with_recursive_bias", # "--strategy-path", -# str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), +# str(Path(__file__).parent.parent / "strategy/strats/recursive_bias"), # "--targeted-trade-amount", # "10", # "--minimum-trade-amount", @@ -68,15 +69,15 @@ def recursive_conf(default_conf_usdt): # pargs['config'] = None # with pytest.raises(OperationalException, # match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): -# start_lookahead_analysis(pargs) +# start_recursive_analysis(pargs) # # Missing timerange # args = [ -# "lookahead-analysis", +# "recursive-analysis", # "--strategy", -# "strategy_test_v3_with_lookahead_bias", +# "strategy_test_v3_with_recursive_bias", # "--strategy-path", -# str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), +# str(Path(__file__).parent.parent / "strategy/strats/recursive_bias"), # "--pairs", # "UNITTEST/BTC", # "--max-open-trades", @@ -86,10 +87,10 @@ def recursive_conf(default_conf_usdt): # pargs['config'] = None # with pytest.raises(OperationalException, # match=r"Please set a timerange\..*"): -# start_lookahead_analysis(pargs) +# start_recursive_analysis(pargs) -# def test_lookahead_helper_invalid_config(recursive_conf) -> None: +# def test_recursive_helper_invalid_config(recursive_conf) -> None: # conf = deepcopy(recursive_conf) # conf['targeted_trade_amount'] = 10 # conf['minimum_trade_amount'] = 40 @@ -98,7 +99,7 @@ def recursive_conf(default_conf_usdt): # RecursiveAnalysisSubFunctions.start(conf) -# def test_lookahead_helper_no_strategy_defined(recursive_conf): +# def test_recursive_helper_no_strategy_defined(recursive_conf): # conf = deepcopy(recursive_conf) # conf['pairs'] = ['UNITTEST/USDT'] # del conf['strategy'] @@ -107,229 +108,72 @@ def recursive_conf(default_conf_usdt): # RecursiveAnalysisSubFunctions.start(conf) -# def test_lookahead_helper_start(recursive_conf, mocker) -> None: -# single_mock = MagicMock() -# text_table_mock = MagicMock() -# mocker.patch.multiple( -# 'freqtrade.optimize.lookahead_analysis_helpers.RecursiveAnalysisSubFunctions', -# initialize_single_lookahead_analysis=single_mock, -# text_table_lookahead_analysis_instances=text_table_mock, -# ) -# RecursiveAnalysisSubFunctions.start(recursive_conf) -# assert single_mock.call_count == 1 -# assert text_table_mock.call_count == 1 +def test_recursive_helper_start(recursive_conf, mocker) -> None: + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', + initialize_single_recursive_analysis=single_mock, + text_table_recursive_analysis_instances=text_table_mock, + ) + RecursiveAnalysisSubFunctions.start(recursive_conf) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 -# single_mock.reset_mock() -# text_table_mock.reset_mock() + single_mock.reset_mock() + text_table_mock.reset_mock() -# def test_lookahead_helper_text_table_lookahead_analysis_instances(recursive_conf): -# analysis = Analysis() -# analysis.has_bias = True -# analysis.total_signals = 5 -# analysis.false_entry_signals = 4 -# analysis.false_exit_signals = 3 +def test_recursive_helper_text_table_recursive_analysis_instances(recursive_conf): + dict_diff = dict() + dict_diff['rsi'] = {} + dict_diff['rsi'][100] = "0.078%" -# strategy_obj = { -# 'name': "strategy_test_v3_with_lookahead_bias", -# 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") -# } + strategy_obj = { + 'name': "strategy_test_v3_recursive_issue", + 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") + } -# instance = LookaheadAnalysis(recursive_conf, strategy_obj) -# instance.current_analysis = analysis -# table, headers, data = (RecursiveAnalysisSubFunctions. -# text_table_lookahead_analysis_instances(recursive_conf, [instance])) + instance = RecursiveAnalysis(recursive_conf, strategy_obj) + instance.dict_recursive = dict_diff + table, headers, data = (RecursiveAnalysisSubFunctions. + text_table_recursive_analysis_instances([instance])) -# # check row contents for a try that has too few signals -# assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' -# assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' -# assert data[0][2].__contains__('too few trades') -# assert len(data[0]) == 3 + # check row contents for a try that has too few signals + assert data[0][0] == 'rsi' + assert data[0][1] == '0.078%' + assert len(data[0]) == 2 -# # now check for an error which occured after enough trades -# analysis.total_signals = 12 -# analysis.false_entry_signals = 11 -# analysis.false_exit_signals = 10 -# instance = LookaheadAnalysis(recursive_conf, strategy_obj) -# instance.current_analysis = analysis -# table, headers, data = (RecursiveAnalysisSubFunctions. -# text_table_lookahead_analysis_instances(recursive_conf, [instance])) -# assert data[0][2].__contains__("error") - -# # edit it into not showing an error -# instance.failed_bias_check = False -# table, headers, data = (RecursiveAnalysisSubFunctions. -# text_table_lookahead_analysis_instances(recursive_conf, [instance])) -# assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' -# assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' -# assert data[0][2] # True -# assert data[0][3] == 12 -# assert data[0][4] == 11 -# assert data[0][5] == 10 -# assert data[0][6] == '' - -# analysis.false_indicators.append('falseIndicator1') -# analysis.false_indicators.append('falseIndicator2') -# table, headers, data = (RecursiveAnalysisSubFunctions. -# text_table_lookahead_analysis_instances(recursive_conf, [instance])) - -# assert data[0][6] == 'falseIndicator1, falseIndicator2' - -# # check amount of returning rows -# assert len(data) == 1 - -# # check amount of multiple rows -# table, headers, data = (RecursiveAnalysisSubFunctions.text_table_lookahead_analysis_instances( -# recursive_conf, [instance, instance, instance])) -# assert len(data) == 3 + # now check when there is no issue + dict_diff = dict() + instance = RecursiveAnalysis(recursive_conf, strategy_obj) + instance.dict_recursive = dict_diff + table, headers, data = (RecursiveAnalysisSubFunctions. + text_table_recursive_analysis_instances([instance])) + assert len(data) == 0 -# def test_lookahead_helper_export_to_csv(recursive_conf): -# import pandas as pd -# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" +def test_initialize_single_recursive_analysis(recursive_conf, mocker, caplog): + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + patch_exchange(mocker) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + recursive_conf['pairs'] = ['UNITTEST/BTC'] -# # just to be sure the test won't fail: remove file if exists for some reason -# # (repeat this at the end once again to clean up) -# if Path(recursive_conf['lookahead_analysis_exportfilename']).exists(): -# Path(recursive_conf['lookahead_analysis_exportfilename']).unlink() + recursive_conf['timeframe'] = '5m' + recursive_conf['timerange'] = '20180119-20180122' + start_mock = mocker.patch('freqtrade.optimize.recursive_analysis.RecursiveAnalysis.start') + strategy_obj = { + 'name': "strategy_test_v3_recursive_issue", + 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") + } -# # before we can start we have to delete the + instance = RecursiveAnalysisSubFunctions.initialize_single_recursive_analysis( + recursive_conf, strategy_obj) + assert log_has_re(r"Recursive test of .* started\.", caplog) + assert start_mock.call_count == 1 -# # 1st check: create a new file and verify its contents -# analysis1 = Analysis() -# analysis1.has_bias = True -# analysis1.total_signals = 12 -# analysis1.false_entry_signals = 11 -# analysis1.false_exit_signals = 10 -# analysis1.false_indicators.append('falseIndicator1') -# analysis1.false_indicators.append('falseIndicator2') -# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" - -# strategy_obj1 = { -# 'name': "strat1", -# 'location': Path("file1.py"), -# } - -# instance1 = LookaheadAnalysis(recursive_conf, strategy_obj1) -# instance1.failed_bias_check = False -# instance1.current_analysis = analysis1 - -# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance1]) -# saved_data1 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) - -# expected_values1 = [ -# [ -# 'file1.py', 'strat1', True, -# 12, 11, 10, -# "falseIndicator1,falseIndicator2" -# ], -# ] -# expected_columns = ['filename', 'strategy', 'has_bias', -# 'total_signals', 'biased_entry_signals', 'biased_exit_signals', -# 'biased_indicators'] -# expected_data1 = pd.DataFrame(expected_values1, columns=expected_columns) - -# assert Path(recursive_conf['lookahead_analysis_exportfilename']).exists() -# assert expected_data1.equals(saved_data1) - -# # 2nd check: update the same strategy (which internally changed or is being retested) -# expected_values2 = [ -# [ -# 'file1.py', 'strat1', False, -# 22, 21, 20, -# "falseIndicator3,falseIndicator4" -# ], -# ] -# expected_data2 = pd.DataFrame(expected_values2, columns=expected_columns) - -# analysis2 = Analysis() -# analysis2.has_bias = False -# analysis2.total_signals = 22 -# analysis2.false_entry_signals = 21 -# analysis2.false_exit_signals = 20 -# analysis2.false_indicators.append('falseIndicator3') -# analysis2.false_indicators.append('falseIndicator4') - -# strategy_obj2 = { -# 'name': "strat1", -# 'location': Path("file1.py"), -# } - -# instance2 = LookaheadAnalysis(recursive_conf, strategy_obj2) -# instance2.failed_bias_check = False -# instance2.current_analysis = analysis2 - -# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance2]) -# saved_data2 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) - -# assert expected_data2.equals(saved_data2) - -# # 3rd check: now we add a new row to an already existing file -# expected_values3 = [ -# [ -# 'file1.py', 'strat1', False, -# 22, 21, 20, -# "falseIndicator3,falseIndicator4" -# ], -# [ -# 'file3.py', 'strat3', True, -# 32, 31, 30, "falseIndicator5,falseIndicator6" -# ], -# ] - -# expected_data3 = pd.DataFrame(expected_values3, columns=expected_columns) - -# analysis3 = Analysis() -# analysis3.has_bias = True -# analysis3.total_signals = 32 -# analysis3.false_entry_signals = 31 -# analysis3.false_exit_signals = 30 -# analysis3.false_indicators.append('falseIndicator5') -# analysis3.false_indicators.append('falseIndicator6') -# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" - -# strategy_obj3 = { -# 'name': "strat3", -# 'location': Path("file3.py"), -# } - -# instance3 = LookaheadAnalysis(recursive_conf, strategy_obj3) -# instance3.failed_bias_check = False -# instance3.current_analysis = analysis3 - -# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance3]) -# saved_data3 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) -# assert expected_data3.equals(saved_data3) - -# # remove csv file after the test is done -# if Path(recursive_conf['lookahead_analysis_exportfilename']).exists(): -# Path(recursive_conf['lookahead_analysis_exportfilename']).unlink() - - -# def test_initialize_single_lookahead_analysis(recursive_conf, mocker, caplog): -# mocker.patch('freqtrade.data.history.get_timerange', get_timerange) -# mocker.patch(f'{EXMS}.get_fee', return_value=0.0) -# mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) -# mocker.patch(f'{EXMS}.get_max_pair_stake_amount', return_value=float('inf')) -# patch_exchange(mocker) -# mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', -# PropertyMock(return_value=['UNITTEST/BTC'])) -# recursive_conf['pairs'] = ['UNITTEST/USDT'] - -# recursive_conf['timeframe'] = '5m' -# recursive_conf['timerange'] = '20180119-20180122' -# start_mock = mocker.patch('freqtrade.optimize.lookahead_analysis.LookaheadAnalysis.start') -# strategy_obj = { -# 'name': "strategy_test_v3_with_lookahead_bias", -# 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") -# } - -# instance = RecursiveAnalysisSubFunctions.initialize_single_lookahead_analysis( -# recursive_conf, strategy_obj) -# assert log_has_re(r"Bias test of .* started\.", caplog) -# assert start_mock.call_count == 1 - -# assert instance.strategy_obj['name'] == "strategy_test_v3_with_lookahead_bias" + assert instance.strategy_obj['name'] == "strategy_test_v3_recursive_issue" @pytest.mark.parametrize('scenario', [ From d92a6d7b737bd8356db9e46a40a29c9c1e9b764e Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 17:51:37 +0900 Subject: [PATCH 31/52] all tests done --- tests/optimize/test_recursive_analysis.py | 123 +++++++++------------- 1 file changed, 47 insertions(+), 76 deletions(-) diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index d06cbdb90..cc7b63f08 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -25,87 +25,58 @@ def recursive_conf(default_conf_usdt): return default_conf_usdt -# def test_start_recursive_analysis(mocker): -# single_mock = MagicMock() -# text_table_mock = MagicMock() -# mocker.patch.multiple( -# 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', -# initialize_single_recursive_analysis=single_mock, -# text_table_recursive_analysis_instances=text_table_mock, -# ) -# args = [ -# "recursive-analysis", -# "--strategy", -# "strategy_test_v3_recursive_issue", -# "--strategy-path", -# str(Path(__file__).parent.parent / "strategy/strats"), -# "--pairs", -# "UNITTEST/BTC", -# "--timerange", -# "20220101-20220201" -# ] -# pargs = get_args(args) -# pargs['config'] = None +def test_start_recursive_analysis(mocker): + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', + initialize_single_recursive_analysis=single_mock, + text_table_recursive_analysis_instances=text_table_mock, + ) + args = [ + "recursive-analysis", + "--strategy", + "strategy_test_v3_recursive_issue", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats"), + "--pairs", + "UNITTEST/BTC", + "--timerange", + "20220101-20220201" + ] + pargs = get_args(args) + pargs['config'] = None -# start_recursive_analysis(pargs) -# assert single_mock.call_count == 1 -# assert text_table_mock.call_count == 1 + start_recursive_analysis(pargs) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 -# single_mock.reset_mock() + single_mock.reset_mock() -# # Test invalid config -# args = [ -# "recursive-analysis", -# "--strategy", -# "strategy_test_v3_with_recursive_bias", -# "--strategy-path", -# str(Path(__file__).parent.parent / "strategy/strats/recursive_bias"), -# "--targeted-trade-amount", -# "10", -# "--minimum-trade-amount", -# "20", -# ] -# pargs = get_args(args) -# pargs['config'] = None -# with pytest.raises(OperationalException, -# match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): -# start_recursive_analysis(pargs) - -# # Missing timerange -# args = [ -# "recursive-analysis", -# "--strategy", -# "strategy_test_v3_with_recursive_bias", -# "--strategy-path", -# str(Path(__file__).parent.parent / "strategy/strats/recursive_bias"), -# "--pairs", -# "UNITTEST/BTC", -# "--max-open-trades", -# "1", -# ] -# pargs = get_args(args) -# pargs['config'] = None -# with pytest.raises(OperationalException, -# match=r"Please set a timerange\..*"): -# start_recursive_analysis(pargs) + # Missing timerange + args = [ + "recursive-analysis", + "--strategy", + "strategy_test_v3_with_recursive_bias", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats"), + "--pairs", + "UNITTEST/BTC" + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"Please set a timerange\..*"): + start_recursive_analysis(pargs) -# def test_recursive_helper_invalid_config(recursive_conf) -> None: -# conf = deepcopy(recursive_conf) -# conf['targeted_trade_amount'] = 10 -# conf['minimum_trade_amount'] = 40 -# with pytest.raises(OperationalException, -# match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): -# RecursiveAnalysisSubFunctions.start(conf) - - -# def test_recursive_helper_no_strategy_defined(recursive_conf): -# conf = deepcopy(recursive_conf) -# conf['pairs'] = ['UNITTEST/USDT'] -# del conf['strategy'] -# with pytest.raises(OperationalException, -# match=r"No Strategy specified"): -# RecursiveAnalysisSubFunctions.start(conf) +def test_recursive_helper_no_strategy_defined(recursive_conf): + conf = deepcopy(recursive_conf) + conf['pairs'] = ['UNITTEST/USDT'] + del conf['strategy'] + with pytest.raises(OperationalException, + match=r"No Strategy specified"): + RecursiveAnalysisSubFunctions.start(conf) def test_recursive_helper_start(recursive_conf, mocker) -> None: From aba576f79f33da4e72c846fec533b56d4c6b0b07 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 17:58:20 +0900 Subject: [PATCH 32/52] pre-commit fix --- tests/optimize/test_recursive_analysis.py | 2 +- tests/strategy/strats/strategy_test_v3_recursive_issue.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index cc7b63f08..6560a4f26 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -10,7 +10,7 @@ from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException from freqtrade.optimize.recursive_analysis import RecursiveAnalysis from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions -from tests.conftest import EXMS, get_args, log_has_re, patch_exchange +from tests.conftest import get_args, log_has_re, patch_exchange @pytest.fixture diff --git a/tests/strategy/strats/strategy_test_v3_recursive_issue.py b/tests/strategy/strats/strategy_test_v3_recursive_issue.py index c4ddc6a1b..78a9dca61 100644 --- a/tests/strategy/strats/strategy_test_v3_recursive_issue.py +++ b/tests/strategy/strats/strategy_test_v3_recursive_issue.py @@ -1,11 +1,10 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement +import talib.abstract as ta from pandas import DataFrame -from technical.indicators import ichimoku from freqtrade.strategy import IStrategy from freqtrade.strategy.parameters import CategoricalParameter -import talib.abstract as ta class strategy_test_v3_recursive_issue(IStrategy): INTERFACE_VERSION = 3 From ce4f1b0709bdf3830672a31dd4f4e36aae7565a4 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 19:56:41 +0900 Subject: [PATCH 33/52] fix test after adding new test strategy --- tests/strategy/test_strategy_loading.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 4cdb35936..a31408b5c 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -35,7 +35,7 @@ def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver._search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 12 + assert len(strategies) == 13 assert isinstance(strategies[0], dict) @@ -43,10 +43,10 @@ def test_search_all_strategies_with_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver._search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 13 + assert len(strategies) == 14 # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 12 + assert len([x for x in strategies if x['class'] is not None]) == 13 assert len([x for x in strategies if x['class'] is None]) == 1 From a0e115ebd92692b8908cbec2719ccc1e14624bb2 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 20:19:39 +0900 Subject: [PATCH 34/52] fix another test --- docs/recursive-analysis.md | 9 ++++++++- tests/commands/test_commands.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 341493f7e..4c102dc14 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -33,9 +33,16 @@ usage: freqtrade recursive-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--freqaimodel-path PATH] [-i TIMEFRAME] [--timerange TIMERANGE] [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}] - [-p PAIRS [PAIRS ...]] + [-p PAIR] [--freqai-backtest-live-models] + [--startup-candle STARTUP_CANDLES [STARTUP_CANDLES ...]] +optional arguments: +-p PAIR, --pairs PAIR + Limit command to this pair. +--startup-candle STARTUP_CANDLE [STARTUP_CANDLE ...] + Provide a space-separated list of startup_candle_count to + be checked. Default : `199 399 499 999 1999`. ``` ### Summary diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index cdaaf9654..e066eac01 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1556,7 +1556,7 @@ def test_start_strategy_updater(mocker, tmpdir): pargs['config'] = None start_strategy_update(pargs) # Number of strategies in the test directory - assert sc_mock.call_count == 11 + assert sc_mock.call_count == 12 sc_mock.reset_mock() args = [ From 478e6f1e6452b3eb120faed432249cd6100ed520 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 21:22:45 +0900 Subject: [PATCH 35/52] fix test --- tests/rpc/test_rpc_apiserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 89cb47830..4bbd07f18 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1636,7 +1636,8 @@ def test_api_strategies(botclient, tmpdir): 'freqai_test_classifier', 'freqai_test_multimodel_classifier_strat', 'freqai_test_multimodel_strat', - 'freqai_test_strat' + 'freqai_test_strat', + 'strategy_test_v3_recursive_issue' ]} From f133ee3cca19a6f3066f1beb08cfbd587842f311 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Fri, 22 Sep 2023 17:10:43 +0900 Subject: [PATCH 36/52] add more explanation in the docs about startup candle and its relation to API limit --- docs/recursive-analysis.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 4c102dc14..6adf5cfe4 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -50,6 +50,15 @@ optional arguments: Checks a given strategy for recursive formula issue via recursive-analysis. Recursive formula issue means that the indicator's calculation don't have enough data for its calculation to produce correct value. +### What's with the odd-numbered default startup candles? +If you look the arguments above, you should notice that the default value for the startup candles are odd number. Why is it? Because when the bot fetchs candle data from exchange's API, the last candle is the one being checked by the bot, and the rest of the data is what we call startup candles. + +For example, Binance allow 1000 candles per API call. When the bot receive 1000 candles, the last candle is being looked at as the current candle, and the 999 candles are the startup candles. If you put the startup candle as 1000 for example, the bot will try to fetch 1001 candles instead. The issue with that is that exchange API send candle data in bulk size. So in case of Binance API, it will send the data in group of 1000. Which means if the bot think the strategy need 1001 data, it will download 2000 data instead, which means you will have 1 current candle and 1999 startup candles. + +Another thing to keep in mind is that Freqtrade only allowed to call the API for 5 times only. So for example, you can only download 5000 data from Binance API, which means the max `startup_candle_count` you can have is 4999. + +Please note that the limit can changed in the future by the exchanges without any prior notice. + ### How does the command work? It will start with a backtest using the supplied timerange to generate a baseline for indicators' value. From 4959d124ad985b49444a545e0ac42183bd7446b7 Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Fri, 22 Sep 2023 10:50:35 +0100 Subject: [PATCH 37/52] Update recursive-analysis.md I went through the docs which are great, but they needed some tidying up and changes to language to meet the style of the rest of the docs. --- docs/recursive-analysis.md | 49 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 6adf5cfe4..14d7bd88a 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -1,26 +1,27 @@ # Recursive analysis -This page explains how to validate your strategy in terms of recursive formula issue. +This page explains how to validate your strategy for inaccuracies due to recursive issues with certain indicators. -First of all, what is recursive formula? Recursive formula is a formula that defines any term of a sequence in terms of its preceding term(s). Example of a recursive formula is an = an-1 + b. +A recursive formula defines any term of a sequence relative to its preceding term(s). An example of a recursive formula is an = an-1 + b. -Second question is why is it matter for Freqtrade? It matters because in backtesting, the bot will get full data of the pairs according to the timerange specified. But in dry/live run, the bot will have limited amounts of data, limited by what each exchanges gives. +Why does this matter for Freqtrade? In backtesting, the bot will get full data of the pairs according to the timerange specified. But in a dry/live run, the bot will be limited by the amount of data each exchanges gives. -For example, let's say that I want to calculate a very basic indicator called `steps`. The first row's value is always 0, while the following rows' values are equal to the value of the previous row's plus 1. If I were to calculate it using latest 1000 candles, then the `steps` value of first row is 0, and the `steps` value at last closed candle is 999. +For example, to calculate a very basic indicator called `steps`, the first row's value is always 0, while the following rows' values are equal to the value of the previous row plus 1. If I were to calculate it using the latest 1000 candles, then the `steps` value of the first row is 0, and the `steps` value at the last closed candle is 999. -But what if I only calculate based of latest 500 candles? Then instead of 999, the `steps` value at last closed candle is 499. The difference of the value means your backtest result can differ from your dry/live run result. +What happens if the calculation is using only the latest 500 candles? Then instead of 999, the `steps` value at last closed candle is 499. The difference of the value means your backtest result can differ from your dry/live run result. -Recursive-analysis requires historic data to be available. To learn how to get data for the pairs and exchange you're interested in, +The `recursive-analysis` command requires historic data to be available. To learn how to get data for the pairs and exchange you're interested in, head over to the [Data Downloading](data-download.md) section of the documentation. -This command is built upon backtesting since it internally chains backtests to prepare different lenghts of data and calculate indicators based of each of the prepared data. -This is done by not looking at the strategy itself - but at the value of the indicators it returned. After multiple backtests are done to calculate the indicators of different startup candles value, the values of last rows are compared to see hoe much differences are they compared to the base backtest. +This command is built upon backtesting since it internally chains backtests to prepare different lengths of data and calculates indicators based on the downloaded data. +This does not run the strategy itself, but rather uses the indicators it contains. After multiple backtests are done to calculate the indicators of different startup candle values (`startup_candle_count`), the values of last rows across all backtests are compared to see how much variance they show compared to the base backtest. -- `--cache` is forced to "none". -- Since we are only looking at indicators' value, using more than one pair is redundant. It is recommended to set the pair used in the command using `-p` flag, preferably using pair with high price, such as BTC or ETH, to avoid having rounding issue that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis is the first pair in the whitelist. -- It's recommended to set a long timerange (at least consist of 5000 candles), so that the initial backtest that going to be used as benchmark have very small or no recursive issue at all. For example, for a 5m timeframe, timerange of 5000 candles would be equal to 18 days. +Command settings: +- Use the `-p` option to set your desired pair to analyse. Since we are only looking at indicator values, using more than one pair is redundant. Preferably use a pair with a relatively high price and at least moderate volatility, such as BTC or ETH, to avoid rounding issues that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis is the first pair in the whitelist. +- It is recommended to set a long timerange (at least 5000 candles) so that the initial backtest that is going to be used as a benchmark has very small or no recursive issues itself. For example, for a 5m timeframe, a timerange of 5000 candles would be equal to 18 days. +- `--cache` is forced to "none" to avoid loading previous backtest results automatically. -Beside recursive formula check, this command also going to do a simple lookahead bias check on the indicators' value only. It won't replace [Lookahead-analysis](lookahead-analysis.md), since this check won't check the difference in trades' entries and exits, which is the important effect of lookahead bias. It will only check whether there is any lookahead bias in indicators if the end of the data are moved. +In addition to the recursive formula check, this command also carries out a simple lookahead bias check on the indicator values only. For a full lookahead check, use [Lookahead-analysis](lookahead-analysis.md). ## Recursive-analysis command reference @@ -47,25 +48,25 @@ optional arguments: ### Summary -Checks a given strategy for recursive formula issue via recursive-analysis. -Recursive formula issue means that the indicator's calculation don't have enough data for its calculation to produce correct value. +Checks a given strategy for recursive formula issue via `recursive-analysis`. -### What's with the odd-numbered default startup candles? -If you look the arguments above, you should notice that the default value for the startup candles are odd number. Why is it? Because when the bot fetchs candle data from exchange's API, the last candle is the one being checked by the bot, and the rest of the data is what we call startup candles. +### Why are odd-numbered default startup candles used? -For example, Binance allow 1000 candles per API call. When the bot receive 1000 candles, the last candle is being looked at as the current candle, and the 999 candles are the startup candles. If you put the startup candle as 1000 for example, the bot will try to fetch 1001 candles instead. The issue with that is that exchange API send candle data in bulk size. So in case of Binance API, it will send the data in group of 1000. Which means if the bot think the strategy need 1001 data, it will download 2000 data instead, which means you will have 1 current candle and 1999 startup candles. +The default value for startup candles are odd numbers. When the bot fetches candle data from the exchange's API, the last candle is the one being checked by the bot and the rest of the data are the "startup candles". -Another thing to keep in mind is that Freqtrade only allowed to call the API for 5 times only. So for example, you can only download 5000 data from Binance API, which means the max `startup_candle_count` you can have is 4999. +For example, Binance allows 1000 candles per API call. When the bot receives 1000 candles, the last candle is the "current candle", and the preceding 999 candles are the "startup candles". By setting the startup candle count as 1000 instead of 999, the bot will try to fetch 1001 candles instead. The exchange API will then send candle data in a paginated form, i.e. in case of the Binance API, this will be two groups- one of length 1000 and another of length 1. This results in the bot thinking the strategy needs 1001 candles of data, and so it will download 2000 candles worth of data instead, which means there will be 1 "current candle" and 1999 "startup candles". -Please note that the limit can changed in the future by the exchanges without any prior notice. +Furthermore, exchanges limit the number of consecutive bulk API calls, e.g. Binance allows 5 calls. In this case, only 5000 candles can be downloaded from Binance API without hitting the API rate limit, which means the max `startup_candle_count` you can have is 4999. + +Please note that this candle limit may be changed in the future by the exchanges without any prior notice. ### How does the command work? -It will start with a backtest using the supplied timerange to generate a baseline for indicators' value. -After setting the baseline it will then do additional runs for each different startup candles. -When the additional runs are done, it will compare the indicators at the last rows and report the differences in a table. +- Firstly an initial backtest is carried out using the supplied timerange to generate a benchmark for indicator values. +- After setting the benchmark it will then carry out additional runs for each different startup candle count. +- It will then compare the indicator values at the last candle rows and report the differences in a table. ### Caveats -- `recursive-analysis` will only calculate and compare the indicators' value at the last row. If there are any differences, the table will only tell you the percentage differences. Whether it has any real impact on your entries and exits isn't checked. -- The ideal scenario is to have your indicators have no difference at all despite the startup candle being varied. But in reality, some of publicly-available formulas are using recursive formula. So the goal isn't to have zero differences, but to have the differences low enough to make sure they won't have any real impact on trading decisions. +- `recursive-analysis` will only calculate and compare the indicator values at the last row. The output table reports the percentage differences between the different startup candle count backtests and the original benchmark backtest. Whether it has any actual impact on your entries and exits is not included. +- The ideal scenario is that indicators will have no variance (or at least very close to 0%) despite the startup candle being varied. In reality, indicators such as EMA are using a recursive formula to calculate indicator values, so the goal is not necessarily to have zero percentage variance, but to have the variance low enough (and the `startup_candle_count` high enough) that the recursion inherent in the indicator will not have any real impact on trading decisions. From 3881c51892a8ef10ed6375b45b8aaf7472fbc967 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Fri, 22 Sep 2023 20:45:48 +0900 Subject: [PATCH 38/52] add example of the result of the analysis --- docs/recursive-analysis.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 14d7bd88a..6c56a319c 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -66,7 +66,26 @@ Please note that this candle limit may be changed in the future by the exchanges - After setting the benchmark it will then carry out additional runs for each different startup candle count. - It will then compare the indicator values at the last candle rows and report the differences in a table. -### Caveats +## Understand the recursive analysis output + +This is an example of how the output will look like when at least one indicator have recursive formula issue + +``` +| indicators | 20 | 40 | 80 | 100 | 150 | 300 | 999 | +|--------------+---------+---------+--------+--------+---------+---------+--------| +| rsi_30 | nan% | -6.025% | 0.612% | 0.828% | -0.140% | 0.000% | 0.000% | +| rsi_14 | 24.141% | -0.876% | 0.070% | 0.007% | -0.000% | -0.000% | - | +``` + +The numbers at the header indicates different `startup_candle_count` used in the analysis. The numbers in the table indicates how much varied are they compared to the benchmark value. + +`nan%` means the value of that indicator can't be calculated due to lack of data. In this example, you can't calculate rsi with length of 30 with just 21 (1 current candle + 20 startup candles) data. + +Important thing to note, we can't tell you which `startup_candle_count` to use because it depends on each users' preference on how much variance is small enough in their opinion to not have any effect on entries and/or exits. + +Aiming for zero variance (shown by `-` value) might not be the best option, because some indicators might requires you to use a very long startup to have zero variance. + +## Caveats - `recursive-analysis` will only calculate and compare the indicator values at the last row. The output table reports the percentage differences between the different startup candle count backtests and the original benchmark backtest. Whether it has any actual impact on your entries and exits is not included. - The ideal scenario is that indicators will have no variance (or at least very close to 0%) despite the startup candle being varied. In reality, indicators such as EMA are using a recursive formula to calculate indicator values, so the goal is not necessarily to have zero percentage variance, but to have the variance low enough (and the `startup_candle_count` high enough) that the recursion inherent in the indicator will not have any real impact on trading decisions. From 30e8466cae4c44f0dff55d4d00f9fa12ab8e73c3 Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Fri, 22 Sep 2023 13:11:35 +0100 Subject: [PATCH 39/52] Update recursive-analysis.md Thanks for adding the example table! It's good to have that example available. I've edited this section of the docs as before. --- docs/recursive-analysis.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 6c56a319c..31642346c 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -66,9 +66,9 @@ Please note that this candle limit may be changed in the future by the exchanges - After setting the benchmark it will then carry out additional runs for each different startup candle count. - It will then compare the indicator values at the last candle rows and report the differences in a table. -## Understand the recursive analysis output +## Understanding the recursive-analysis output -This is an example of how the output will look like when at least one indicator have recursive formula issue +This is an example of an output results table where at least one indicator has a recursive formula issue: ``` | indicators | 20 | 40 | 80 | 100 | 150 | 300 | 999 | @@ -77,13 +77,13 @@ This is an example of how the output will look like when at least one indicator | rsi_14 | 24.141% | -0.876% | 0.070% | 0.007% | -0.000% | -0.000% | - | ``` -The numbers at the header indicates different `startup_candle_count` used in the analysis. The numbers in the table indicates how much varied are they compared to the benchmark value. +The column headers indicate the different `startup_candle_count` used in the analysis. The values in the table indicate the variance of the backtested indicators compared to the benchmark value. -`nan%` means the value of that indicator can't be calculated due to lack of data. In this example, you can't calculate rsi with length of 30 with just 21 (1 current candle + 20 startup candles) data. +`nan%` means the value of that indicator cannot be calculated due to lack of data. In this example, you cannot calculate RSI with length 30 with just 21 candles (1 current candle + 20 startup candles). -Important thing to note, we can't tell you which `startup_candle_count` to use because it depends on each users' preference on how much variance is small enough in their opinion to not have any effect on entries and/or exits. +Users should assess the table per indicator to decide if the specified `startup_candle_count` results in a sufficiently small variance so that the indicator does not have any effect on entries and/or exits. -Aiming for zero variance (shown by `-` value) might not be the best option, because some indicators might requires you to use a very long startup to have zero variance. +As such, aiming for absolute zero variance (shown by `-` value) might not be the best option, because some indicators might require you to use such a long `startup_candle_count` to have zero variance. ## Caveats From 9fa365e7660cf76a35d23a771dafedd0c7b793e4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Sep 2023 10:47:30 +0200 Subject: [PATCH 40/52] Fix docs rendering (lists need a newline before the first item) --- docs/recursive-analysis.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 31642346c..ba0823c7e 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -17,7 +17,8 @@ This command is built upon backtesting since it internally chains backtests to p This does not run the strategy itself, but rather uses the indicators it contains. After multiple backtests are done to calculate the indicators of different startup candle values (`startup_candle_count`), the values of last rows across all backtests are compared to see how much variance they show compared to the base backtest. Command settings: -- Use the `-p` option to set your desired pair to analyse. Since we are only looking at indicator values, using more than one pair is redundant. Preferably use a pair with a relatively high price and at least moderate volatility, such as BTC or ETH, to avoid rounding issues that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis is the first pair in the whitelist. + +- Use the `-p` option to set your desired pair to analyze. Since we are only looking at indicator values, using more than one pair is redundant. Preferably use a pair with a relatively high price and at least moderate volatility, such as BTC or ETH, to avoid rounding issues that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis is the first pair in the whitelist. - It is recommended to set a long timerange (at least 5000 candles) so that the initial backtest that is going to be used as a benchmark has very small or no recursive issues itself. For example, for a 5m timeframe, a timerange of 5000 candles would be equal to 18 days. - `--cache` is forced to "none" to avoid loading previous backtest results automatically. From 6cfc1836a242f6d677333f2fcd582a02e5582a40 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 23 Sep 2023 19:23:03 +0900 Subject: [PATCH 41/52] fix wrong startup candle --- freqtrade/templates/sample_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index 65a6e440e..dec547715 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -77,7 +77,7 @@ class SampleStrategy(IStrategy): exit_short_rsi = IntParameter(low=1, high=50, default=30, space='buy', optimize=True, load=True) # Number of candles the strategy requires before producing valid signals - startup_candle_count: int = 170 + startup_candle_count: int = 200 # Optional order type mapping. order_types = { From c0b2b0b96dc9057af8b1cd651c57d7995514a892 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 23 Sep 2023 19:27:57 +0900 Subject: [PATCH 42/52] fix args description --- freqtrade/commands/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index f7c17b09c..736ae49fc 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -474,7 +474,7 @@ class Arguments: # Add recursive_analysis subcommand recursive_analayis_cmd = subparsers.add_parser( 'recursive-analysis', - help="Check for potential look ahead bias.", + help="Check for potential recursive formula issue.", parents=[_common_parser, _strategy_parser]) recursive_analayis_cmd.set_defaults(func=start_recursive_analysis) From aac9ee0d55e3093c9a760f2abdfb24e310ae6d7f Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 24 Sep 2023 14:28:18 +0900 Subject: [PATCH 43/52] move lookahead and recursive docs under Advanced Topics --- mkdocs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index bb5ae0010..f15d9ca18 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,7 +22,6 @@ nav: - Web Hook: webhook-config.md - Data Downloading: data-download.md - Backtesting: backtesting.md - - Lookahead analysis: lookahead-analysis.md - Hyperopt: hyperopt.md - FreqAI: - Introduction: freqai.md @@ -45,6 +44,8 @@ nav: - Trade Object: trade-object.md - Advanced Strategy: strategy-advanced.md - Advanced Hyperopt: advanced-hyperopt.md + - Lookahead analysis: lookahead-analysis.md + - Recursive analysis: recursive-analysis.md - Producer/Consumer mode: producer-consumer.md - Edge Positioning: edge.md - FAQ: faq.md From 3c647a3794910ae684117c87288ea603052e08ed Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 24 Sep 2023 14:55:13 +0900 Subject: [PATCH 44/52] change wordings on the doc from backtest to "calculate indicators" since we never do full backtest process --- docs/recursive-analysis.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index ba0823c7e..dd939a30a 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -13,14 +13,14 @@ What happens if the calculation is using only the latest 500 candles? Then inste The `recursive-analysis` command requires historic data to be available. To learn how to get data for the pairs and exchange you're interested in, head over to the [Data Downloading](data-download.md) section of the documentation. -This command is built upon backtesting since it internally chains backtests to prepare different lengths of data and calculates indicators based on the downloaded data. -This does not run the strategy itself, but rather uses the indicators it contains. After multiple backtests are done to calculate the indicators of different startup candle values (`startup_candle_count`), the values of last rows across all backtests are compared to see how much variance they show compared to the base backtest. +This command is built upon preparing different lengths of data and calculates indicators based on them. +This does not backtest the strategy itself, but rather only calculates the indicators. After calculating the indicators of different startup candle values (`startup_candle_count`) are done, the values of last rows across all specified `startup_candle_count` are compared to see how much variance they show compared to the base calculation. Command settings: - Use the `-p` option to set your desired pair to analyze. Since we are only looking at indicator values, using more than one pair is redundant. Preferably use a pair with a relatively high price and at least moderate volatility, such as BTC or ETH, to avoid rounding issues that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis is the first pair in the whitelist. -- It is recommended to set a long timerange (at least 5000 candles) so that the initial backtest that is going to be used as a benchmark has very small or no recursive issues itself. For example, for a 5m timeframe, a timerange of 5000 candles would be equal to 18 days. -- `--cache` is forced to "none" to avoid loading previous backtest results automatically. +- It is recommended to set a long timerange (at least 5000 candles) so that the initial indicators' calculation that is going to be used as a benchmark has very small or no recursive issues itself. For example, for a 5m timeframe, a timerange of 5000 candles would be equal to 18 days. +- `--cache` is forced to "none" to avoid loading previous indicators calculation automatically. In addition to the recursive formula check, this command also carries out a simple lookahead bias check on the indicator values only. For a full lookahead check, use [Lookahead-analysis](lookahead-analysis.md). @@ -63,7 +63,7 @@ Please note that this candle limit may be changed in the future by the exchanges ### How does the command work? -- Firstly an initial backtest is carried out using the supplied timerange to generate a benchmark for indicator values. +- Firstly an initial indicator calculation is carried out using the supplied timerange to generate a benchmark for indicator values. - After setting the benchmark it will then carry out additional runs for each different startup candle count. - It will then compare the indicator values at the last candle rows and report the differences in a table. @@ -78,7 +78,7 @@ This is an example of an output results table where at least one indicator has a | rsi_14 | 24.141% | -0.876% | 0.070% | 0.007% | -0.000% | -0.000% | - | ``` -The column headers indicate the different `startup_candle_count` used in the analysis. The values in the table indicate the variance of the backtested indicators compared to the benchmark value. +The column headers indicate the different `startup_candle_count` used in the analysis. The values in the table indicate the variance of the calculated indicators compared to the benchmark value. `nan%` means the value of that indicator cannot be calculated due to lack of data. In this example, you cannot calculate RSI with length 30 with just 21 candles (1 current candle + 20 startup candles). @@ -88,5 +88,6 @@ As such, aiming for absolute zero variance (shown by `-` value) might not be the ## Caveats -- `recursive-analysis` will only calculate and compare the indicator values at the last row. The output table reports the percentage differences between the different startup candle count backtests and the original benchmark backtest. Whether it has any actual impact on your entries and exits is not included. +- `recursive-analysis` will only calculate and compare the indicator values at the last row. The output table reports the percentage differences between the different startup candle count calculations and the original benchmark calculation. Whether it has any actual impact on your entries and exits is not included. - The ideal scenario is that indicators will have no variance (or at least very close to 0%) despite the startup candle being varied. In reality, indicators such as EMA are using a recursive formula to calculate indicator values, so the goal is not necessarily to have zero percentage variance, but to have the variance low enough (and the `startup_candle_count` high enough) that the recursion inherent in the indicator will not have any real impact on trading decisions. +- `recursive-analysis` will only run calculations on `populate_indicators` and `@informative` decorator(s). If you put any indicator calculation on `populate_entry_trend` or `populate_exit_trend`, it won't be calculated \ No newline at end of file From b387c315da2548b07c510533ffaf75698786394a Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 24 Sep 2023 14:59:19 +0900 Subject: [PATCH 45/52] fix pre-commit --- docs/recursive-analysis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index dd939a30a..c197a3e67 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -90,4 +90,4 @@ As such, aiming for absolute zero variance (shown by `-` value) might not be the - `recursive-analysis` will only calculate and compare the indicator values at the last row. The output table reports the percentage differences between the different startup candle count calculations and the original benchmark calculation. Whether it has any actual impact on your entries and exits is not included. - The ideal scenario is that indicators will have no variance (or at least very close to 0%) despite the startup candle being varied. In reality, indicators such as EMA are using a recursive formula to calculate indicator values, so the goal is not necessarily to have zero percentage variance, but to have the variance low enough (and the `startup_candle_count` high enough) that the recursion inherent in the indicator will not have any real impact on trading decisions. -- `recursive-analysis` will only run calculations on `populate_indicators` and `@informative` decorator(s). If you put any indicator calculation on `populate_entry_trend` or `populate_exit_trend`, it won't be calculated \ No newline at end of file +- `recursive-analysis` will only run calculations on `populate_indicators` and `@informative` decorator(s). If you put any indicator calculation on `populate_entry_trend` or `populate_exit_trend`, it won't be calculated From b6885266239be7544d315cf2f604e35d00653771 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Sep 2023 20:48:47 +0200 Subject: [PATCH 46/52] Improve sorting in docs (these are more important than advanced strategy/docs) --- mkdocs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index f15d9ca18..53894d007 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,10 +42,10 @@ nav: - Advanced Topics: - Advanced Post-installation Tasks: advanced-setup.md - Trade Object: trade-object.md - - Advanced Strategy: strategy-advanced.md - - Advanced Hyperopt: advanced-hyperopt.md - Lookahead analysis: lookahead-analysis.md - Recursive analysis: recursive-analysis.md + - Advanced Strategy: strategy-advanced.md + - Advanced Hyperopt: advanced-hyperopt.md - Producer/Consumer mode: producer-consumer.md - Edge Positioning: edge.md - FAQ: faq.md From 37550d3bdb4b96b3d9a3c0529b971e78f8128c18 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Sep 2023 21:01:37 +0200 Subject: [PATCH 47/52] Fix typo in --strategy_list --- freqtrade/optimize/lookahead_analysis_helpers.py | 6 +++--- freqtrade/optimize/recursive_analysis_helpers.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 422026780..81ea5d61b 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -184,12 +184,12 @@ class LookaheadAnalysisSubFunctions: lookaheadAnalysis_instances = [] - # unify --strategy and --strategy_list to one list + # unify --strategy and --strategy-list to one list if not (strategy_list := config.get('strategy_list', [])): if config.get('strategy') is None: raise OperationalException( "No Strategy specified. Please specify a strategy via --strategy or " - "--strategy_list" + "--strategy-list" ) strategy_list = [config['strategy']] @@ -211,5 +211,5 @@ class LookaheadAnalysisSubFunctions: else: logger.error("There were no strategies specified neither through " "--strategy nor through " - "--strategy_list " + "--strategy-list " "or timeframe was not specified.") diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py index b33a24cf4..2bf2dfe2a 100644 --- a/freqtrade/optimize/recursive_analysis_helpers.py +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -77,12 +77,12 @@ class RecursiveAnalysisSubFunctions: RecursiveAnalysis_instances = [] - # unify --strategy and --strategy_list to one list + # unify --strategy and --strategy-list to one list if not (strategy_list := config.get('strategy_list', [])): if config.get('strategy') is None: raise OperationalException( "No Strategy specified. Please specify a strategy via --strategy or " - "--strategy_list" + "--strategy-list" ) strategy_list = [config['strategy']] @@ -102,5 +102,5 @@ class RecursiveAnalysisSubFunctions: else: logger.error("There were no strategies specified neither through " "--strategy nor through " - "--strategy_list " + "--strategy-list " "or timeframe was not specified.") From 39ede449a06b7e64bd0b2b1bcb1c7c9041842c35 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Sep 2023 06:58:16 +0200 Subject: [PATCH 48/52] Rename test to avoid naming collision --- tests/optimize/test_recursive_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 6560a4f26..52dfd8ac6 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -150,7 +150,7 @@ def test_initialize_single_recursive_analysis(recursive_conf, mocker, caplog): @pytest.mark.parametrize('scenario', [ 'no_bias', 'bias1' ]) -def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: +def test_recursive_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: mocker.patch('freqtrade.data.history.get_timerange', get_timerange) patch_exchange(mocker) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', From 20ea679b2b3deafaab0c4c71c40565da12a6ee06 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Sep 2023 07:06:11 +0200 Subject: [PATCH 49/52] Add "bias2" test with full lookahead bias --- tests/optimize/test_recursive_analysis.py | 19 +++++++++++-------- .../strategy_test_v3_recursive_issue.py | 6 ++++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 52dfd8ac6..0492ed5e0 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -148,7 +148,7 @@ def test_initialize_single_recursive_analysis(recursive_conf, mocker, caplog): @pytest.mark.parametrize('scenario', [ - 'no_bias', 'bias1' + 'no_bias', 'bias1', 'bias2' ]) def test_recursive_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: mocker.patch('freqtrade.data.history.get_timerange', get_timerange) @@ -177,10 +177,13 @@ def test_recursive_biased_strategy(recursive_conf, mocker, caplog, scenario) -> # Assert init correct assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog) - diff_pct = abs(float(instance.dict_recursive['rsi'][100].replace("%", ""))) - # check non-biased strategy - if scenario == "no_bias": - assert diff_pct < 0.01 - # check biased strategy - elif scenario == "bias1": - assert diff_pct >= 0.01 + if scenario == "bias2": + assert log_has_re("=> found lookahead in indicator rsi", caplog) + else: + diff_pct = abs(float(instance.dict_recursive['rsi'][100].replace("%", ""))) + # check non-biased strategy + if scenario == "no_bias": + assert diff_pct < 0.01 + # check biased strategy + elif scenario == "bias1": + assert diff_pct >= 0.01 diff --git a/tests/strategy/strats/strategy_test_v3_recursive_issue.py b/tests/strategy/strats/strategy_test_v3_recursive_issue.py index 78a9dca61..974c2c1c2 100644 --- a/tests/strategy/strats/strategy_test_v3_recursive_issue.py +++ b/tests/strategy/strats/strategy_test_v3_recursive_issue.py @@ -19,7 +19,7 @@ class strategy_test_v3_recursive_issue(IStrategy): # Optimal timeframe for the strategy timeframe = '5m' - scenario = CategoricalParameter(['no_bias', 'bias1'], default='bias1', space="buy") + scenario = CategoricalParameter(['no_bias', 'bias1', 'bias2'], default='bias1', space="buy") # Number of candles the strategy requires before producing valid signals startup_candle_count: int = 100 @@ -28,8 +28,10 @@ class strategy_test_v3_recursive_issue(IStrategy): # bias is introduced here if self.scenario.value == 'no_bias': dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) - else: + elif self.scenario.value == 'bias1': dataframe['rsi'] = ta.RSI(dataframe, timeperiod=50) + else: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=50).shift(-1) return dataframe From 92d7f279833f2925587fd96665295be6ad03b447 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Sep 2023 07:09:48 +0200 Subject: [PATCH 50/52] Further update test a bit --- tests/optimize/test_recursive_analysis.py | 15 +++++++-------- .../strats/strategy_test_v3_recursive_issue.py | 8 +++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 0492ed5e0..f025f5b76 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -179,11 +179,10 @@ def test_recursive_biased_strategy(recursive_conf, mocker, caplog, scenario) -> if scenario == "bias2": assert log_has_re("=> found lookahead in indicator rsi", caplog) - else: - diff_pct = abs(float(instance.dict_recursive['rsi'][100].replace("%", ""))) - # check non-biased strategy - if scenario == "no_bias": - assert diff_pct < 0.01 - # check biased strategy - elif scenario == "bias1": - assert diff_pct >= 0.01 + diff_pct = abs(float(instance.dict_recursive['rsi'][100].replace("%", ""))) + # check non-biased strategy + if scenario == "no_bias": + assert diff_pct < 0.01 + # check biased strategy + elif scenario in ("bias1", "bias2"): + assert diff_pct >= 0.01 diff --git a/tests/strategy/strats/strategy_test_v3_recursive_issue.py b/tests/strategy/strats/strategy_test_v3_recursive_issue.py index 974c2c1c2..b3074113d 100644 --- a/tests/strategy/strats/strategy_test_v3_recursive_issue.py +++ b/tests/strategy/strats/strategy_test_v3_recursive_issue.py @@ -28,10 +28,12 @@ class strategy_test_v3_recursive_issue(IStrategy): # bias is introduced here if self.scenario.value == 'no_bias': dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) - elif self.scenario.value == 'bias1': - dataframe['rsi'] = ta.RSI(dataframe, timeperiod=50) else: - dataframe['rsi'] = ta.RSI(dataframe, timeperiod=50).shift(-1) + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=50) + + if self.scenario.value == 'bias2': + # Has both bias1 and bias2 + dataframe['rsi_lookahead'] = ta.RSI(dataframe, timeperiod=50).shift(-1) return dataframe From 154149ff036560849dc03729fee5c7be76688877 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Sep 2023 07:14:49 +0200 Subject: [PATCH 51/52] Slightly updated docs --- docs/recursive-analysis.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index c197a3e67..512b79f8f 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -47,10 +47,6 @@ optional arguments: be checked. Default : `199 399 499 999 1999`. ``` -### Summary - -Checks a given strategy for recursive formula issue via `recursive-analysis`. - ### Why are odd-numbered default startup candles used? The default value for startup candles are odd numbers. When the bot fetches candle data from the exchange's API, the last candle is the one being checked by the bot and the rest of the data are the "startup candles". @@ -64,8 +60,8 @@ Please note that this candle limit may be changed in the future by the exchanges ### How does the command work? - Firstly an initial indicator calculation is carried out using the supplied timerange to generate a benchmark for indicator values. -- After setting the benchmark it will then carry out additional runs for each different startup candle count. -- It will then compare the indicator values at the last candle rows and report the differences in a table. +- After setting the benchmark it will then carry out additional runs for each of the different startup candle count values. +- The command will then compare the indicator values at the last candle rows and report the differences in a table. ## Understanding the recursive-analysis output @@ -89,5 +85,5 @@ As such, aiming for absolute zero variance (shown by `-` value) might not be the ## Caveats - `recursive-analysis` will only calculate and compare the indicator values at the last row. The output table reports the percentage differences between the different startup candle count calculations and the original benchmark calculation. Whether it has any actual impact on your entries and exits is not included. -- The ideal scenario is that indicators will have no variance (or at least very close to 0%) despite the startup candle being varied. In reality, indicators such as EMA are using a recursive formula to calculate indicator values, so the goal is not necessarily to have zero percentage variance, but to have the variance low enough (and the `startup_candle_count` high enough) that the recursion inherent in the indicator will not have any real impact on trading decisions. -- `recursive-analysis` will only run calculations on `populate_indicators` and `@informative` decorator(s). If you put any indicator calculation on `populate_entry_trend` or `populate_exit_trend`, it won't be calculated +- The ideal scenario is that indicators will have no variance (or at least very close to 0%) despite the startup candle being varied. In reality, indicators such as EMA are using a recursive formula to calculate indicator values, so the goal is not necessarily to have zero percentage variance, but to have the variance low enough (and therefore `startup_candle_count` high enough) that the recursion inherent in the indicator will not have any real impact on trading decisions. +- `recursive-analysis` will only run calculations on `populate_indicators` and `@informative` decorator(s). If you put any indicator calculation on `populate_entry_trend` or `populate_exit_trend`, it won't be calculated. From 7971cb29bbb33412b1cba6c4f9d5394a52c1828a Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Fri, 29 Sep 2023 14:17:44 +0900 Subject: [PATCH 52/52] fix error message --- freqtrade/optimize/recursive_analysis_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py index 2bf2dfe2a..ba1cf8745 100644 --- a/freqtrade/optimize/recursive_analysis_helpers.py +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -42,7 +42,7 @@ class RecursiveAnalysisSubFunctions: # setting a timerange is enforced here raise OperationalException( "Please set a timerange. " - "A timerange of 20 candles are enough for recursive analysis." + "A timerange of 5000 candles are enough for recursive analysis." ) if config.get('backtest_cache') is None: