From b1dbc3a65f3bdab8b1f2ad7fcc127f7de1d42569 Mon Sep 17 00:00:00 2001 From: Wagner Costa Santos Date: Thu, 22 Sep 2022 12:13:51 -0300 Subject: [PATCH 1/2] remove function remove_training_from_backtesting and ensure BT period is correct with startup_candle_count --- freqtrade/data/dataprovider.py | 12 +++++++++--- freqtrade/freqai/data_kitchen.py | 23 ----------------------- freqtrade/optimize/backtesting.py | 9 +++++++-- 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 43850ddd9..51cacc2c5 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -90,8 +90,12 @@ class DataProvider: if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) + # It is not necessary to add the training candles, as they + # were already added at the beginning of the backtest. + add_train_candles = False + # Move informative start time respecting startup_candle_count - startup_candles = self.get_required_startup(str(timeframe)) + startup_candles = self.get_required_startup(str(timeframe), add_train_candles) tf_seconds = timeframe_to_seconds(str(timeframe)) timerange.subtract_start(tf_seconds * startup_candles) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( @@ -105,7 +109,7 @@ class DataProvider: ) return self.__cached_pairs_backtesting[saved_pair].copy() - def get_required_startup(self, timeframe: str) -> int: + def get_required_startup(self, timeframe: str, add_train_candles: bool = True) -> int: freqai_config = self._config.get('freqai', {}) if not freqai_config.get('enabled', False): return self._config.get('startup_candle_count', 0) @@ -115,7 +119,9 @@ class DataProvider: # make sure the startupcandles is at least the set maximum indicator periods self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods)) tf_seconds = timeframe_to_seconds(timeframe) - train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds + train_candles = 0 + if add_train_candles: + train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds total_candles = int(self._config['startup_candle_count'] + train_candles) logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') return total_candles diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index d2abd0ad2..dd2ef3bad 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -466,27 +466,6 @@ class FreqaiDataKitchen: return df - def remove_training_from_backtesting( - self - ) -> DataFrame: - """ - Function which takes the backtesting time range and - remove training data from dataframe, keeping only the - startup_candle_count candles - """ - startup_candle_count = self.config.get('startup_candle_count', 0) - tf = self.config['timeframe'] - tr = self.config["timerange"] - - backtesting_timerange = TimeRange.parse_timerange(tr) - if startup_candle_count > 0 and backtesting_timerange: - backtesting_timerange.subtract_start(timeframe_to_seconds(tf) * startup_candle_count) - - start = datetime.fromtimestamp(backtesting_timerange.startts, tz=timezone.utc) - df = self.return_dataframe - df = df.loc[df["date"] >= start, :] - return df - def principal_component_analysis(self) -> None: """ Performs Principal Component Analysis on the data for dimensionality reduction @@ -979,8 +958,6 @@ class FreqaiDataKitchen: to_keep = [col for col in dataframe.columns if not col.startswith("&")] self.return_dataframe = pd.concat([dataframe[to_keep], self.full_df], axis=1) - - self.return_dataframe = self.remove_training_from_backtesting() self.full_df = DataFrame() return diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0a05d740d..20429eb97 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -139,9 +139,14 @@ class Backtesting: # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) + self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) + + if self.config.get('freqai', {}).get('enabled', False): + # For FreqAI, increase the required_startup to includes the training data + self.required_startup = self.dataprovider.get_required_startup(self.timeframe) + # Add maximum startup candle count to configuration for informative pairs support self.config['startup_candle_count'] = self.required_startup - self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) self.trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) # strategies which define "can_short=True" will fail to load in Spot mode. @@ -217,7 +222,7 @@ class Backtesting: pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=self.timerange, - startup_candles=self.dataprovider.get_required_startup(self.timeframe), + startup_candles=self.config['startup_candle_count'], fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), candle_type=self.config.get('candle_type_def', CandleType.SPOT) From 166ae8e3a1cd7c2169dd41a534f409dba7e0845e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Sep 2022 15:51:20 +0200 Subject: [PATCH 2/2] Remove missleading comment --- freqtrade/data/dataprovider.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 51cacc2c5..ac3b61d1d 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -90,12 +90,10 @@ class DataProvider: if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) + # It is not necessary to add the training candles, as they # were already added at the beginning of the backtest. - add_train_candles = False - - # Move informative start time respecting startup_candle_count - startup_candles = self.get_required_startup(str(timeframe), add_train_candles) + startup_candles = self.get_required_startup(str(timeframe), False) tf_seconds = timeframe_to_seconds(str(timeframe)) timerange.subtract_start(tf_seconds * startup_candles) self.__cached_pairs_backtesting[saved_pair] = load_pair_history(