diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 127193a35..ecdb2e109 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -371,17 +371,6 @@ class FreqaiDataKitchen: return df - def add_noise_to_training_features(self) -> None: - """ - Add noise to train features to reduce the risk of overfitting. - """ - mu = 0 # no shift - sigma = self.freqai_config["feature_parameters"]["noise_standard_deviation"] - compute_df = self.data_dictionary['train_features'] - noise = np.random.normal(mu, sigma, [compute_df.shape[0], compute_df.shape[1]]) - self.data_dictionary['train_features'] += noise - return - def find_features(self, dataframe: DataFrame) -> None: """ Find features in the strategy provided dataframe diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index ffe0ee8c3..632266b00 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -510,12 +510,12 @@ class IFreqaiModel(ABC): def define_data_pipeline(self, dk: FreqaiDataKitchen) -> None: ft_params = self.freqai_info["feature_parameters"] dk.feature_pipeline = Pipeline([ - ('const', ds.DataSieveVarianceThreshold(threshold=0)), + ('const', ds.VarianceThreshold(threshold=0)), ('scaler', SKLearnWrapper(MinMaxScaler(feature_range=(-1, 1)))) ]) if ft_params.get("principal_component_analysis", False): - dk.feature_pipeline.append(('pca', ds.DataSievePCA())) + dk.feature_pipeline.append(('pca', ds.PCA())) dk.feature_pipeline.append(('post-pca-scaler', SKLearnWrapper(MinMaxScaler(feature_range=(-1, 1))))) @@ -529,14 +529,15 @@ class IFreqaiModel(ABC): dk.feature_pipeline.append(('di', ds.DissimilarityIndex(di_threshold=di))) if ft_params.get("use_DBSCAN_to_remove_outliers", False): - dk.feature_pipeline.append(('dbscan', ds.DataSieveDBSCAN())) + dk.feature_pipeline.append(('dbscan', ds.DBSCAN())) + + sigma = self.freqai_info["feature_parameters"].get('noise_standard_deviation', 0) + if sigma: + dk.feature_pipeline.append(('noise', ds.Noise(sigma=sigma))) dk.feature_pipeline.fitparams = dk.feature_pipeline._validate_fitparams( {}, dk.feature_pipeline.steps) - # if self.freqai_info["feature_parameters"].get('noise_standard_deviation', 0): - # dk.pipeline.extend(('noise', ds.Noise())) - def define_label_pipeline(self, dk: FreqaiDataKitchen) -> None: dk.label_pipeline = Pipeline([ diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 748950e24..a515ba2b5 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -10,4 +10,4 @@ catboost==1.2; 'arm' not in platform_machine and (sys_platform != 'darwin' or py lightgbm==3.3.5 xgboost==1.7.5 tensorboard==2.13.0 -datasieve==0.1.1 +datasieve==0.1.2 diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 90959ec2c..fec9de724 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -37,21 +37,22 @@ def can_run_model(model: str) -> None: pytest.skip("Reinforcement learning / PyTorch module not available on intel based Mac OS.") -@pytest.mark.parametrize('model, pca, dbscan, float32, can_short, shuffle, buffer', [ - ('LightGBMRegressor', True, False, True, True, False, 0), - ('XGBoostRegressor', False, True, False, True, False, 10), - ('XGBoostRFRegressor', False, False, False, True, False, 0), - ('CatboostRegressor', False, False, False, True, True, 0), - ('PyTorchMLPRegressor', False, False, False, False, False, 0), - ('PyTorchTransformerRegressor', False, False, False, False, False, 0), - ('ReinforcementLearner', False, True, False, True, False, 0), - ('ReinforcementLearner_multiproc', False, False, False, True, False, 0), - ('ReinforcementLearner_test_3ac', False, False, False, False, False, 0), - ('ReinforcementLearner_test_3ac', False, False, False, True, False, 0), - ('ReinforcementLearner_test_4ac', False, False, False, True, False, 0), +@pytest.mark.parametrize('model, pca, dbscan, float32, can_short, shuffle, buffer, noise', [ + ('LightGBMRegressor', True, False, True, True, False, 0, 0), + ('XGBoostRegressor', False, True, False, True, False, 10, 0.05), + ('XGBoostRFRegressor', False, False, False, True, False, 0, 0), + ('CatboostRegressor', False, False, False, True, True, 0, 0), + ('PyTorchMLPRegressor', False, False, False, False, False, 0, 0), + ('PyTorchTransformerRegressor', False, False, False, False, False, 0, 0), + ('ReinforcementLearner', False, True, False, True, False, 0, 0), + ('ReinforcementLearner_multiproc', False, False, False, True, False, 0, 0), + ('ReinforcementLearner_test_3ac', False, False, False, False, False, 0, 0), + ('ReinforcementLearner_test_3ac', False, False, False, True, False, 0, 0), + ('ReinforcementLearner_test_4ac', False, False, False, True, False, 0, 0), ]) def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, - dbscan, float32, can_short, shuffle, buffer): + dbscan, float32, can_short, shuffle, + buffer, noise): can_run_model(model) @@ -68,6 +69,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, freqai_conf.update({"reduce_df_footprint": float32}) freqai_conf['freqai']['feature_parameters'].update({"shuffle_after_split": shuffle}) freqai_conf['freqai']['feature_parameters'].update({"buffer_train_data_candles": buffer}) + freqai_conf['freqai']['feature_parameters'].update({"noise_standard_deviation": noise}) if 'ReinforcementLearner' in model: model_save_ext = 'zip'