From 0be115de9c7d9fb5a0568fef7a8894c2458eaf20 Mon Sep 17 00:00:00 2001 From: Wagner Costa Santos Date: Tue, 27 Sep 2022 10:26:57 -0300 Subject: [PATCH] backtest_live_models - added new tests and refactoring --- freqtrade/freqai/data_kitchen.py | 4 +- freqtrade/freqai/freqai_util.py | 65 ++++++++++--------- tests/freqai/test_freqai_util.py | 105 +++++++++++++++++++++++++++++++ tests/test_configuration.py | 72 +++++++++++++++++++++ 4 files changed, 216 insertions(+), 30 deletions(-) create mode 100644 tests/freqai/test_freqai_util.py diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 9a4101dce..7f32c942d 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -473,8 +473,8 @@ class FreqaiDataKitchen: pair_data = self.backtest_live_models_data["pairs_end_dates"][pair] model_end_dates = [] backtesting_timerange = self.backtest_live_models_data["backtesting_timerange"] - for data in pair_data: - model_end_dates.append(data["model_end_date"]) + for end_date in pair_data: + model_end_dates.append(end_date) model_end_dates.append(backtesting_timerange.stopts) model_end_dates.sort() for index, item in enumerate(model_end_dates): diff --git a/freqtrade/freqai/freqai_util.py b/freqtrade/freqai/freqai_util.py index 0d3056b8d..665310230 100644 --- a/freqtrade/freqai/freqai_util.py +++ b/freqtrade/freqai/freqai_util.py @@ -36,34 +36,11 @@ def get_timerange_from_ready_models(models_path: Path) -> Tuple[TimeRange, str, pairs_end_dates: Dict with pair and model end training dates info) """ all_models_end_dates = [] - pairs_end_dates: Dict[str, Any] = {} - if not models_path.is_dir(): - raise OperationalException( - 'Model folders not found. Saved models are required ' - 'to run backtest with the freqai-backtest-live-models option' - ) - for model_dir in models_path.iterdir(): - if str(model_dir.name).startswith("sub-train"): - model_end_date = int(model_dir.name.split("_")[1]) - pair = model_dir.name.split("_")[0].replace("sub-train-", "") - model_file_name = ( - f"cb_{str(model_dir.name).replace('sub-train-', '').lower()}" - "_model.joblib" - ) - - model_path_file = Path(model_dir / model_file_name) - if model_path_file.is_file(): - if pair not in pairs_end_dates: - pairs_end_dates[pair] = [] - - pairs_end_dates[pair].append({ - "model_end_date": model_end_date, - "model_path_file": model_path_file, - "model_dir": model_dir - }) - - if model_end_date not in all_models_end_dates: - all_models_end_dates.append(model_end_date) + pairs_end_dates: Dict[str, Any] = get_pairs_timestamps_training_from_ready_models(models_path) + for key in pairs_end_dates: + for model_end_date in pairs_end_dates[key]: + if model_end_date not in all_models_end_dates: + all_models_end_dates.append(model_end_date) if len(all_models_end_dates) == 0: raise OperationalException( @@ -104,3 +81,35 @@ def get_timerange_from_ready_models(models_path: Path) -> Tuple[TimeRange, str, 'date', 'date', min(all_models_end_dates), max(all_models_end_dates) ) return backtesting_timerange, backtesting_string_timerange, pairs_end_dates + + +def get_pairs_timestamps_training_from_ready_models(models_path: Path) -> Dict[str, Any]: + """ + Scan the models path and returns all pairs end training dates (timestamp) + :param models_path: FreqAI model path + + :returns: + :pairs_end_dates: Dict with pair and model end training dates info + """ + pairs_end_dates: Dict[str, Any] = {} + if not models_path.is_dir(): + raise OperationalException( + 'Model folders not found. Saved models are required ' + 'to run backtest with the freqai-backtest-live-models option' + ) + for model_dir in models_path.iterdir(): + if str(model_dir.name).startswith("sub-train"): + model_end_date = int(model_dir.name.split("_")[1]) + pair = model_dir.name.split("_")[0].replace("sub-train-", "") + model_file_name = ( + f"cb_{str(model_dir.name).replace('sub-train-', '').lower()}" + "_model.joblib" + ) + + model_path_file = Path(model_dir / model_file_name) + if model_path_file.is_file(): + if pair not in pairs_end_dates: + pairs_end_dates[pair] = [] + + pairs_end_dates[pair].append(model_end_date) + return pairs_end_dates diff --git a/tests/freqai/test_freqai_util.py b/tests/freqai/test_freqai_util.py new file mode 100644 index 000000000..2c7c8c68a --- /dev/null +++ b/tests/freqai/test_freqai_util.py @@ -0,0 +1,105 @@ +import platform +from unittest.mock import MagicMock + +import pytest + +from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider +from freqtrade.exceptions import OperationalException +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.freqai.freqai_util import (get_full_model_path, + get_pairs_timestamps_training_from_ready_models, + get_timerange_from_ready_models) +from tests.conftest import get_patched_exchange +from tests.freqai.conftest import get_patched_freqai_strategy + + +def is_arm() -> bool: + machine = platform.machine() + return "arm" in machine or "aarch64" in machine + + +@pytest.mark.parametrize('model', [ + 'LightGBMRegressor' + ]) +def test_get_full_model_path(mocker, freqai_conf, model): + if is_arm() and model == 'CatboostRegressor': + pytest.skip("CatBoost is not supported on ARM") + + freqai_conf.update({"freqaimodel": model}) + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.update({"strategy": "freqai_test_strat"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.extract_data_and_train_model( + new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + model_path = get_full_model_path(freqai_conf) + assert model_path.is_dir() is True + + +def test_get_pairs_timestamp_validation(mocker, freqai_conf): + model_path = get_full_model_path(freqai_conf) + with pytest.raises( + OperationalException, + match=r'.*required to run backtest with the freqai-backtest-live-models.*' + ): + get_pairs_timestamps_training_from_ready_models(model_path) + + +@pytest.mark.parametrize('model', [ + 'LightGBMRegressor' + ]) +def test_get_timerange_from_ready_models(mocker, freqai_conf, model): + if is_arm() and model == 'CatboostRegressor': + pytest.skip("CatBoost is not supported on ARM") + + freqai_conf.update({"freqaimodel": model}) + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.update({"strategy": "freqai_test_strat"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180101-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180101-20180130") + + new_timerange = TimeRange.parse_timerange("20180120-20180122") + freqai.extract_data_and_train_model( + new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + new_timerange = TimeRange.parse_timerange("20180122-20180124") + freqai.extract_data_and_train_model( + new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + model_path = get_full_model_path(freqai_conf) + (backtesting_timerange, + backtesting_string_timerange, + pairs_end_dates) = get_timerange_from_ready_models(models_path=model_path) + + assert len(pairs_end_dates["ADA"]) == 2 + assert backtesting_string_timerange == '20180122-20180127' + assert backtesting_timerange.startts == 1516579200 + assert backtesting_timerange.stopts == 1516924800 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 99edf0233..94d4b4c78 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1574,3 +1574,75 @@ def test_flat_vars_to_nested_dict(caplog): assert log_has("Loading variable 'FREQTRADE__EXCHANGE__SOME_SETTING'", caplog) assert not log_has("Loading variable 'NOT_RELEVANT'", caplog) + + +def test_setup_hyperopt_freqai(mocker, default_conf, caplog) -> None: + patched_configuration_load_config_file(mocker, default_conf) + mocker.patch( + 'freqtrade.configuration.configuration.create_datadir', + lambda c, x: x + ) + mocker.patch( + 'freqtrade.configuration.configuration.create_userdata_dir', + lambda x, *args, **kwargs: Path(x) + ) + arglist = [ + 'hyperopt', + '--config', 'config.json', + '--strategy', CURRENT_TEST_STRATEGY, + '--timerange', '20220801-20220805', + "--freqaimodel", + "LightGBMRegressorMultiTarget", + "--analyze-per-epoch" + ] + + args = Arguments(arglist).get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + config['freqai'] = { + "enabled": True + } + with pytest.raises( + OperationalException, match=r".*analyze-per-epoch parameter is not supported.*" + ): + validate_config_consistency(config) + + +def test_setup_freqai_backtest_live_models(mocker, default_conf, caplog) -> None: + patched_configuration_load_config_file(mocker, default_conf) + mocker.patch( + 'freqtrade.configuration.configuration.create_datadir', + lambda c, x: x + ) + mocker.patch( + 'freqtrade.configuration.configuration.create_userdata_dir', + lambda x, *args, **kwargs: Path(x) + ) + arglist = [ + 'backtesting', + '--config', 'config.json', + '--strategy', CURRENT_TEST_STRATEGY, + '--timerange', '20220801-20220805', + "--freqaimodel", + "LightGBMRegressorMultiTarget", + "--freqai-backtest-live-models" + ] + + args = Arguments(arglist).get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + with pytest.raises( + OperationalException, match=r".*--freqai-backtest-live-models parameter is only.*" + ): + validate_config_consistency(config) + + conf = deepcopy(config) + conf['freqai'] = { + "enabled": True + } + with pytest.raises( + OperationalException, match=r".* timerange parameter is not supported with .*" + ): + validate_config_consistency(conf)