ruff format: freqai tests

This commit is contained in:
Matthias 2024-05-12 16:02:21 +02:00
parent ffd49e0e59
commit 40e161a5b9
4 changed files with 256 additions and 211 deletions

View File

@ -29,26 +29,34 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c
patch_exchange(mocker)
now = datetime.now(timezone.utc)
mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
mocker.patch('freqtrade.optimize.backtesting.history.load_data')
mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now))
mocker.patch(
"freqtrade.plugins.pairlistmanager.PairListManager.whitelist",
PropertyMock(return_value=["HULUMULU/USDT", "XRP/USDT"]),
)
mocker.patch("freqtrade.optimize.backtesting.history.load_data")
mocker.patch("freqtrade.optimize.backtesting.history.get_timerange", return_value=(now, now))
patched_configuration_load_config_file(mocker, freqai_conf)
args = [
'backtesting',
'--config', 'config.json',
'--datadir', str(testdatadir),
'--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'),
'--timeframe', '1m',
'--strategy-list', CURRENT_TEST_STRATEGY
"backtesting",
"--config",
"config.json",
"--datadir",
str(testdatadir),
"--strategy-path",
str(Path(__file__).parents[1] / "strategy/strats"),
"--timeframe",
"1m",
"--strategy-list",
CURRENT_TEST_STRATEGY,
]
args = get_args(args)
bt_config = setup_optimize_configuration(args, RunMode.BACKTEST)
Backtesting(bt_config)
assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical',
caplog)
assert log_has_re(
"Using --strategy-list with FreqAI REQUIRES all strategies to have identical", caplog
)
Backtesting.cleanup()
@ -60,23 +68,29 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c
("1d", 302),
],
)
def test_freqai_backtest_load_data(freqai_conf, mocker, caplog,
timeframe, expected_startup_candle_count):
def test_freqai_backtest_load_data(
freqai_conf, mocker, caplog, timeframe, expected_startup_candle_count
):
patch_exchange(mocker)
now = datetime.now(timezone.utc)
mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
mocker.patch('freqtrade.optimize.backtesting.history.load_data')
mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now))
freqai_conf['timeframe'] = timeframe
freqai_conf.get('freqai', {}).get('feature_parameters', {}).update({'include_timeframes': []})
mocker.patch(
"freqtrade.plugins.pairlistmanager.PairListManager.whitelist",
PropertyMock(return_value=["HULUMULU/USDT", "XRP/USDT"]),
)
mocker.patch("freqtrade.optimize.backtesting.history.load_data")
mocker.patch("freqtrade.optimize.backtesting.history.get_timerange", return_value=(now, now))
freqai_conf["timeframe"] = timeframe
freqai_conf.get("freqai", {}).get("feature_parameters", {}).update({"include_timeframes": []})
backtesting = Backtesting(deepcopy(freqai_conf))
backtesting.load_bt_data()
assert log_has_re(f'Increasing startup_candle_count for freqai on {timeframe} '
f'to {expected_startup_candle_count}', caplog)
assert history.load_data.call_args[1]['startup_candles'] == expected_startup_candle_count
assert log_has_re(
f"Increasing startup_candle_count for freqai on {timeframe} "
f"to {expected_startup_candle_count}",
caplog,
)
assert history.load_data.call_args[1]["startup_candles"] == expected_startup_candle_count
Backtesting.cleanup()
@ -85,45 +99,55 @@ def test_freqai_backtest_live_models_model_not_found(freqai_conf, mocker, testda
patch_exchange(mocker)
now = datetime.now(timezone.utc)
mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
mocker.patch('freqtrade.optimize.backtesting.history.load_data')
mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now))
mocker.patch(
"freqtrade.plugins.pairlistmanager.PairListManager.whitelist",
PropertyMock(return_value=["HULUMULU/USDT", "XRP/USDT"]),
)
mocker.patch("freqtrade.optimize.backtesting.history.load_data")
mocker.patch("freqtrade.optimize.backtesting.history.get_timerange", return_value=(now, now))
freqai_conf["timerange"] = ""
freqai_conf.get("freqai", {}).update({"backtest_using_historic_predictions": False})
patched_configuration_load_config_file(mocker, freqai_conf)
args = [
'backtesting',
'--config', 'config.json',
'--datadir', str(testdatadir),
'--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'),
'--timeframe', '5m',
'--freqai-backtest-live-models'
"backtesting",
"--config",
"config.json",
"--datadir",
str(testdatadir),
"--strategy-path",
str(Path(__file__).parents[1] / "strategy/strats"),
"--timeframe",
"5m",
"--freqai-backtest-live-models",
]
args = get_args(args)
bt_config = setup_optimize_configuration(args, RunMode.BACKTEST)
with pytest.raises(OperationalException,
match=r".* Historic predictions data is required to run backtest .*"):
with pytest.raises(
OperationalException, match=r".* Historic predictions data is required to run backtest .*"
):
Backtesting(bt_config)
Backtesting.cleanup()
def test_freqai_backtest_consistent_timerange(mocker, freqai_conf):
freqai_conf['runmode'] = 'backtest'
mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
PropertyMock(return_value=['XRP/USDT:USDT']))
freqai_conf["runmode"] = "backtest"
mocker.patch(
"freqtrade.plugins.pairlistmanager.PairListManager.whitelist",
PropertyMock(return_value=["XRP/USDT:USDT"]),
)
gbs = mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats')
gbs = mocker.patch("freqtrade.optimize.backtesting.generate_backtest_stats")
freqai_conf['candle_type_def'] = CandleType.FUTURES
freqai_conf.get('exchange', {}).update({'pair_whitelist': ['XRP/USDT:USDT']})
freqai_conf.get('freqai', {}).get('feature_parameters', {}).update(
{'include_timeframes': ['5m', '1h'], 'include_corr_pairlist': []})
freqai_conf['timerange'] = '20211120-20211121'
freqai_conf["candle_type_def"] = CandleType.FUTURES
freqai_conf.get("exchange", {}).update({"pair_whitelist": ["XRP/USDT:USDT"]})
freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(
{"include_timeframes": ["5m", "1h"], "include_corr_pairlist": []}
)
freqai_conf["timerange"] = "20211120-20211121"
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
@ -139,6 +163,6 @@ def test_freqai_backtest_consistent_timerange(mocker, freqai_conf):
backtesting = Backtesting(deepcopy(freqai_conf))
backtesting.start()
assert gbs.call_args[1]['min_date'] == datetime(2021, 11, 20, 0, 0, tzinfo=timezone.utc)
assert gbs.call_args[1]['max_date'] == datetime(2021, 11, 21, 0, 0, tzinfo=timezone.utc)
assert gbs.call_args[1]["min_date"] == datetime(2021, 11, 20, 0, 0, tzinfo=timezone.utc)
assert gbs.call_args[1]["max_date"] == datetime(2021, 11, 21, 0, 0, tzinfo=timezone.utc)
Backtesting.cleanup()

View File

@ -1,4 +1,3 @@
import shutil
from pathlib import Path
from unittest.mock import patch
@ -15,7 +14,7 @@ from tests.freqai.conftest import get_patched_freqai_strategy
def test_update_historic_data(mocker, freqai_conf):
freqai_conf['runmode'] = 'backtest'
freqai_conf["runmode"] = "backtest"
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
@ -99,7 +98,7 @@ def test_use_strategy_to_populate_indicators(mocker, freqai_conf):
sub_timerange = TimeRange.parse_timerange("20180111-20180114")
corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk)
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC')
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
assert len(df.columns) == 33
shutil.rmtree(Path(freqai.dk.full_path))
@ -133,10 +132,7 @@ def test_get_timerange_from_backtesting_live_df_pred_not_found(mocker, freqai_co
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
freqai = strategy.freqai
with pytest.raises(
OperationalException,
match=r'Historic predictions not found.*'
):
with pytest.raises(OperationalException, match=r"Historic predictions not found.*"):
freqai.dd.get_timerange_from_live_historic_predictions()
@ -158,13 +154,10 @@ def test_set_initial_return_values(mocker, freqai_conf):
start_x_plus_1 = "2023-08-30"
end_x_plus_5 = "2023-09-03"
historic_data = {
'date_pred': pd.date_range(end=end_x, periods=5),
'value': range(1, 6)
}
historic_data = {"date_pred": pd.date_range(end=end_x, periods=5), "value": range(1, 6)}
new_data = {
'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5),
'value': range(6, 11)
"date": pd.date_range(start=start_x_plus_1, end=end_x_plus_5),
"value": range(6, 11),
}
freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data)
@ -173,20 +166,21 @@ def test_set_initial_return_values(mocker, freqai_conf):
dataframe = pd.DataFrame(new_data)
# Action
with patch('logging.Logger.warning') as mock_logger_warning:
with patch("logging.Logger.warning") as mock_logger_warning:
freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe)
# Assertions
hist_pred_df = freqai.dd.historic_predictions[pair]
model_return_df = freqai.dd.model_return_values[pair]
assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5)
assert 'date_pred' in hist_pred_df.columns
assert hist_pred_df["date_pred"].iloc[-1] == pd.Timestamp(end_x_plus_5)
assert "date_pred" in hist_pred_df.columns
assert hist_pred_df.shape[0] == 8
# compare values in model_return_df with hist_pred_df
assert (model_return_df["value"].values ==
hist_pred_df.tail(len(dataframe))["value"].values).all()
assert (
model_return_df["value"].values == hist_pred_df.tail(len(dataframe))["value"].values
).all()
assert model_return_df.shape[0] == len(dataframe)
# Ensure logger error is not called
@ -212,13 +206,10 @@ def test_set_initial_return_values_warning(mocker, freqai_conf):
start_x_plus_1 = "2023-09-01"
end_x_plus_5 = "2023-09-05"
historic_data = {
'date_pred': pd.date_range(end=end_x, periods=5),
'value': range(1, 6)
}
historic_data = {"date_pred": pd.date_range(end=end_x, periods=5), "value": range(1, 6)}
new_data = {
'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5),
'value': range(6, 11)
"date": pd.date_range(start=start_x_plus_1, end=end_x_plus_5),
"value": range(6, 11),
}
freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data)
@ -227,20 +218,21 @@ def test_set_initial_return_values_warning(mocker, freqai_conf):
dataframe = pd.DataFrame(new_data)
# Action
with patch('logging.Logger.warning') as mock_logger_warning:
with patch("logging.Logger.warning") as mock_logger_warning:
freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe)
# Assertions
hist_pred_df = freqai.dd.historic_predictions[pair]
model_return_df = freqai.dd.model_return_values[pair]
assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5)
assert 'date_pred' in hist_pred_df.columns
assert hist_pred_df["date_pred"].iloc[-1] == pd.Timestamp(end_x_plus_5)
assert "date_pred" in hist_pred_df.columns
assert hist_pred_df.shape[0] == 10
# compare values in model_return_df with hist_pred_df
assert (model_return_df["value"].values == hist_pred_df.tail(
len(dataframe))["value"].values).all()
assert (
model_return_df["value"].values == hist_pred_df.tail(len(dataframe))["value"].values
).all()
assert model_return_df.shape[0] == len(dataframe)
# Ensure logger error is not called

View File

@ -67,7 +67,6 @@ def test_split_timerange(
def test_check_if_model_expired(mocker, freqai_conf):
dk = get_patched_data_kitchen(mocker, freqai_conf)
now = datetime.now(tz=timezone.utc).timestamp()
assert dk.check_if_model_expired(now) is False
@ -105,12 +104,10 @@ def test_make_train_test_datasets(mocker, freqai_conf):
assert data_dictionary
assert len(data_dictionary) == 7
assert len(data_dictionary['train_features'].index) == 1916
assert len(data_dictionary["train_features"].index) == 1916
@pytest.mark.parametrize('model', [
'LightGBMRegressor'
])
@pytest.mark.parametrize("model", ["LightGBMRegressor"])
def test_get_full_model_path(mocker, freqai_conf, model):
freqai_conf.update({"freqaimodel": model})
freqai_conf.update({"timerange": "20180110-20180130"})
@ -134,9 +131,10 @@ def test_get_full_model_path(mocker, freqai_conf, model):
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
new_timerange = TimeRange.parse_timerange("20180120-20180130")
freqai.dk.set_paths('ADA/BTC', None)
freqai.dk.set_paths("ADA/BTC", None)
freqai.extract_data_and_train_model(
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange
)
model_path = freqai.dk.get_full_models_path(freqai_conf)
assert model_path.is_dir() is True
@ -161,7 +159,7 @@ def test_get_pair_data_for_features_with_prealoaded_data(mocker, freqai_conf):
def test_get_pair_data_for_features_without_preloaded_data(mocker, freqai_conf):
freqai_conf.update({"timerange": "20180115-20180130"})
freqai_conf['runmode'] = 'backtest'
freqai_conf["runmode"] = "backtest"
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
@ -172,13 +170,13 @@ def test_get_pair_data_for_features_without_preloaded_data(mocker, freqai_conf):
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
base_df = {'5m': pd.DataFrame()}
base_df = {"5m": pd.DataFrame()}
df = freqai.dk.get_pair_data_for_features("LTC/BTC", "5m", strategy, base_dataframes=base_df)
assert df is not base_df["5m"]
assert not df.empty
assert df.iloc[0]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-11 23:00:00"
assert df.iloc[-1]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-30 00:00:00"
assert df.iloc[0]["date"].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-11 23:00:00"
assert df.iloc[-1]["date"].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-30 00:00:00"
def test_populate_features(mocker, freqai_conf):
@ -192,12 +190,14 @@ def test_populate_features(mocker, freqai_conf):
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(timerange, "LTC/BTC", freqai.dk)
mocker.patch.object(strategy, 'feature_engineering_expand_all', return_value=base_df["5m"])
df = freqai.dk.populate_features(base_df["5m"], "LTC/BTC", strategy,
base_dataframes=base_df, corr_dataframes=corr_df)
mocker.patch.object(strategy, "feature_engineering_expand_all", return_value=base_df["5m"])
df = freqai.dk.populate_features(
base_df["5m"], "LTC/BTC", strategy, base_dataframes=base_df, corr_dataframes=corr_df
)
strategy.feature_engineering_expand_all.assert_called_once()
pd.testing.assert_frame_equal(base_df["5m"],
strategy.feature_engineering_expand_all.call_args[0][0])
pd.testing.assert_frame_equal(
base_df["5m"], strategy.feature_engineering_expand_all.call_args[0][0]
)
assert df.iloc[0]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-15 00:00:00"
assert df.iloc[0]["date"].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-15 00:00:00"

View File

@ -24,7 +24,7 @@ from tests.freqai.conftest import (
def can_run_model(model: str) -> None:
is_pytorch_model = 'Reinforcement' in model or 'PyTorch' in model
is_pytorch_model = "Reinforcement" in model or "PyTorch" in model
if is_arm() and "Catboost" in model:
pytest.skip("CatBoost is not supported on ARM.")
@ -33,57 +33,59 @@ def can_run_model(model: str) -> None:
pytest.skip("Reinforcement learning / PyTorch module not available on intel based Mac OS.")
@pytest.mark.parametrize('model, pca, dbscan, float32, can_short, shuffle, buffer, noise', [
('LightGBMRegressor', True, False, True, True, False, 0, 0),
('XGBoostRegressor', False, True, False, True, False, 10, 0.05),
('XGBoostRFRegressor', False, False, False, True, False, 0, 0),
('CatboostRegressor', False, False, False, True, True, 0, 0),
('PyTorchMLPRegressor', False, False, False, False, False, 0, 0),
('PyTorchTransformerRegressor', False, False, False, False, False, 0, 0),
('ReinforcementLearner', False, True, False, True, False, 0, 0),
('ReinforcementLearner_multiproc', False, False, False, True, False, 0, 0),
('ReinforcementLearner_test_3ac', False, False, False, False, False, 0, 0),
('ReinforcementLearner_test_3ac', False, False, False, True, False, 0, 0),
('ReinforcementLearner_test_4ac', False, False, False, True, False, 0, 0),
])
def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca,
dbscan, float32, can_short, shuffle,
buffer, noise):
@pytest.mark.parametrize(
"model, pca, dbscan, float32, can_short, shuffle, buffer, noise",
[
("LightGBMRegressor", True, False, True, True, False, 0, 0),
("XGBoostRegressor", False, True, False, True, False, 10, 0.05),
("XGBoostRFRegressor", False, False, False, True, False, 0, 0),
("CatboostRegressor", False, False, False, True, True, 0, 0),
("PyTorchMLPRegressor", False, False, False, False, False, 0, 0),
("PyTorchTransformerRegressor", False, False, False, False, False, 0, 0),
("ReinforcementLearner", False, True, False, True, False, 0, 0),
("ReinforcementLearner_multiproc", False, False, False, True, False, 0, 0),
("ReinforcementLearner_test_3ac", False, False, False, False, False, 0, 0),
("ReinforcementLearner_test_3ac", False, False, False, True, False, 0, 0),
("ReinforcementLearner_test_4ac", False, False, False, True, False, 0, 0),
],
)
def test_extract_data_and_train_model_Standard(
mocker, freqai_conf, model, pca, dbscan, float32, can_short, shuffle, buffer, noise
):
can_run_model(model)
test_tb = True
if is_mac():
test_tb = False
model_save_ext = 'joblib'
model_save_ext = "joblib"
freqai_conf.update({"freqaimodel": model})
freqai_conf.update({"timerange": "20180110-20180130"})
freqai_conf.update({"strategy": "freqai_test_strat"})
freqai_conf['freqai']['feature_parameters'].update({"principal_component_analysis": pca})
freqai_conf['freqai']['feature_parameters'].update({"use_DBSCAN_to_remove_outliers": dbscan})
freqai_conf["freqai"]["feature_parameters"].update({"principal_component_analysis": pca})
freqai_conf["freqai"]["feature_parameters"].update({"use_DBSCAN_to_remove_outliers": dbscan})
freqai_conf.update({"reduce_df_footprint": float32})
freqai_conf['freqai']['feature_parameters'].update({"shuffle_after_split": shuffle})
freqai_conf['freqai']['feature_parameters'].update({"buffer_train_data_candles": buffer})
freqai_conf['freqai']['feature_parameters'].update({"noise_standard_deviation": noise})
freqai_conf["freqai"]["feature_parameters"].update({"shuffle_after_split": shuffle})
freqai_conf["freqai"]["feature_parameters"].update({"buffer_train_data_candles": buffer})
freqai_conf["freqai"]["feature_parameters"].update({"noise_standard_deviation": noise})
if 'ReinforcementLearner' in model:
model_save_ext = 'zip'
if "ReinforcementLearner" in model:
model_save_ext = "zip"
freqai_conf = make_rl_config(freqai_conf)
# test the RL guardrails
freqai_conf['freqai']['feature_parameters'].update({"use_SVM_to_remove_outliers": True})
freqai_conf['freqai']['feature_parameters'].update({"DI_threshold": 2})
freqai_conf['freqai']['data_split_parameters'].update({'shuffle': True})
freqai_conf["freqai"]["feature_parameters"].update({"use_SVM_to_remove_outliers": True})
freqai_conf["freqai"]["feature_parameters"].update({"DI_threshold": 2})
freqai_conf["freqai"]["data_split_parameters"].update({"shuffle": True})
if 'test_3ac' in model or 'test_4ac' in model:
if "test_3ac" in model or "test_4ac" in model:
freqai_conf["freqaimodel_path"] = str(Path(__file__).parents[1] / "freqai" / "test_models")
freqai_conf["freqai"]["rl_config"]["drop_ohlc_from_features"] = True
if 'PyTorch' in model:
model_save_ext = 'zip'
if "PyTorch" in model:
model_save_ext = "zip"
pytorch_mlp_mtp = mock_pytorch_mlp_model_training_parameters()
freqai_conf['freqai']['model_training_parameters'].update(pytorch_mlp_mtp)
if 'Transformer' in model:
freqai_conf["freqai"]["model_training_parameters"].update(pytorch_mlp_mtp)
if "Transformer" in model:
# transformer model takes a window, unlike the MLP regressor
freqai_conf.update({"conv_width": 10})
@ -97,7 +99,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca,
freqai.can_short = can_short
freqai.dk = FreqaiDataKitchen(freqai_conf)
freqai.dk.live = True
freqai.dk.set_paths('ADA/BTC', 10000)
freqai.dk.set_paths("ADA/BTC", 10000)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
@ -105,32 +107,37 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca,
data_load_timerange = TimeRange.parse_timerange("20180125-20180130")
new_timerange = TimeRange.parse_timerange("20180127-20180130")
freqai.dk.set_paths('ADA/BTC', None)
freqai.dk.set_paths("ADA/BTC", None)
freqai.train_timer("start", "ADA/BTC")
freqai.extract_data_and_train_model(
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange
)
freqai.train_timer("stop", "ADA/BTC")
freqai.dd.save_metric_tracker_to_disk()
freqai.dd.save_drawer_to_disk()
assert Path(freqai.dk.full_path / "metric_tracker.json").is_file()
assert Path(freqai.dk.full_path / "pair_dictionary.json").is_file()
assert Path(freqai.dk.data_path /
f"{freqai.dk.model_filename}_model.{model_save_ext}").is_file()
assert Path(
freqai.dk.data_path / f"{freqai.dk.model_filename}_model.{model_save_ext}"
).is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file()
shutil.rmtree(Path(freqai.dk.full_path))
@pytest.mark.parametrize('model, strat', [
('LightGBMRegressorMultiTarget', "freqai_test_multimodel_strat"),
('XGBoostRegressorMultiTarget', "freqai_test_multimodel_strat"),
('CatboostRegressorMultiTarget', "freqai_test_multimodel_strat"),
('LightGBMClassifierMultiTarget', "freqai_test_multimodel_classifier_strat"),
('CatboostClassifierMultiTarget', "freqai_test_multimodel_classifier_strat")
])
@pytest.mark.parametrize(
"model, strat",
[
("LightGBMRegressorMultiTarget", "freqai_test_multimodel_strat"),
("XGBoostRegressorMultiTarget", "freqai_test_multimodel_strat"),
("CatboostRegressorMultiTarget", "freqai_test_multimodel_strat"),
("LightGBMClassifierMultiTarget", "freqai_test_multimodel_classifier_strat"),
("CatboostClassifierMultiTarget", "freqai_test_multimodel_classifier_strat"),
],
)
def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, strat):
can_run_model(model)
@ -152,28 +159,32 @@ def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, s
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
new_timerange = TimeRange.parse_timerange("20180120-20180130")
freqai.dk.set_paths('ADA/BTC', None)
freqai.dk.set_paths("ADA/BTC", None)
freqai.extract_data_and_train_model(
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange
)
assert len(freqai.dk.label_list) == 2
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file()
assert len(freqai.dk.data['training_features_list']) == 14
assert len(freqai.dk.data["training_features_list"]) == 14
shutil.rmtree(Path(freqai.dk.full_path))
@pytest.mark.parametrize('model', [
'LightGBMClassifier',
'CatboostClassifier',
'XGBoostClassifier',
'XGBoostRFClassifier',
'SKLearnRandomForestClassifier',
'PyTorchMLPClassifier',
])
@pytest.mark.parametrize(
"model",
[
"LightGBMClassifier",
"CatboostClassifier",
"XGBoostClassifier",
"XGBoostRFClassifier",
"SKLearnRandomForestClassifier",
"PyTorchMLPClassifier",
],
)
def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model):
can_run_model(model)
@ -196,25 +207,28 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model):
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
new_timerange = TimeRange.parse_timerange("20180120-20180130")
freqai.dk.set_paths('ADA/BTC', None)
freqai.dk.set_paths("ADA/BTC", None)
freqai.extract_data_and_train_model(new_timerange, "ADA/BTC",
strategy, freqai.dk, data_load_timerange)
freqai.extract_data_and_train_model(
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange
)
if 'PyTorchMLPClassifier':
if "PyTorchMLPClassifier":
pytorch_mlp_mtp = mock_pytorch_mlp_model_training_parameters()
freqai_conf['freqai']['model_training_parameters'].update(pytorch_mlp_mtp)
freqai_conf["freqai"]["model_training_parameters"].update(pytorch_mlp_mtp)
if freqai.dd.model_type == 'joblib':
if freqai.dd.model_type == "joblib":
model_file_extension = ".joblib"
elif freqai.dd.model_type == "pytorch":
model_file_extension = ".zip"
else:
raise Exception(f"Unsupported model type: {freqai.dd.model_type},"
f" can't assign model_file_extension")
raise Exception(
f"Unsupported model type: {freqai.dd.model_type}," f" can't assign model_file_extension"
)
assert Path(freqai.dk.data_path /
f"{freqai.dk.model_filename}_model{model_file_extension}").exists()
assert Path(
freqai.dk.data_path / f"{freqai.dk.model_filename}_model{model_file_extension}"
).exists()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").exists()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").exists()
@ -233,7 +247,7 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model):
("XGBoostClassifier", 2, "freqai_test_classifier"),
("LightGBMClassifier", 2, "freqai_test_classifier"),
("CatboostClassifier", 2, "freqai_test_classifier"),
("PyTorchMLPClassifier", 2, "freqai_test_classifier")
("PyTorchMLPClassifier", 2, "freqai_test_classifier"),
],
)
def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog):
@ -243,7 +257,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog)
test_tb = False
freqai_conf.get("freqai", {}).update({"save_backtest_models": True})
freqai_conf['runmode'] = RunMode.BACKTEST
freqai_conf["runmode"] = RunMode.BACKTEST
Trade.use_db = False
@ -251,21 +265,22 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog)
freqai_conf.update({"timerange": "20180120-20180130"})
freqai_conf.update({"strategy": strat})
if 'ReinforcementLearner' in model:
if "ReinforcementLearner" in model:
freqai_conf = make_rl_config(freqai_conf)
if 'test_4ac' in model:
if "test_4ac" in model:
freqai_conf["freqaimodel_path"] = str(Path(__file__).parents[1] / "freqai" / "test_models")
if 'PyTorch' in model:
if "PyTorch" in model:
pytorch_mlp_mtp = mock_pytorch_mlp_model_training_parameters()
freqai_conf['freqai']['model_training_parameters'].update(pytorch_mlp_mtp)
if 'Transformer' in model:
freqai_conf["freqai"]["model_training_parameters"].update(pytorch_mlp_mtp)
if "Transformer" in model:
# transformer model takes a window, unlike the MLP regressor
freqai_conf.update({"conv_width": 10})
freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(
{"indicator_periods_candles": [2]})
{"indicator_periods_candles": [2]}
)
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
@ -282,7 +297,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog)
df = base_df[freqai_conf["timeframe"]]
metadata = {"pair": "LTC/BTC"}
freqai.dk.set_paths('LTC/BTC', None)
freqai.dk.set_paths("LTC/BTC", None)
freqai.start_backtesting(df, metadata, freqai.dk, strategy)
model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
@ -294,13 +309,16 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog)
def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf):
freqai_conf.update({"timerange": "20180120-20180124"})
freqai_conf['runmode'] = 'backtest'
freqai_conf.get("freqai", {}).update({
freqai_conf["runmode"] = "backtest"
freqai_conf.get("freqai", {}).update(
{
"backtest_period_days": 0.5,
"save_backtest_models": True,
})
}
)
freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(
{"indicator_periods_candles": [2]})
{"indicator_periods_candles": [2]}
)
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
@ -325,10 +343,11 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf):
def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
freqai_conf.update({"timerange": "20180120-20180130"})
freqai_conf['runmode'] = 'backtest'
freqai_conf["runmode"] = "backtest"
freqai_conf.get("freqai", {}).update({"save_backtest_models": True})
freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(
{"indicator_periods_candles": [2]})
{"indicator_periods_candles": [2]}
)
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
@ -381,7 +400,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
freqai.dk.pair = pair
freqai.start_backtesting(df, metadata, freqai.dk, strategy)
path = (freqai.dd.full_path / freqai.dk.backtest_predictions_folder)
path = freqai.dd.full_path / freqai.dk.backtest_predictions_folder
prediction_files = [x for x in path.iterdir() if x.is_file()]
assert len(prediction_files) == 2
@ -389,7 +408,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
def test_backtesting_fit_live_predictions(mocker, freqai_conf, caplog):
freqai_conf['runmode'] = 'backtest'
freqai_conf["runmode"] = "backtest"
freqai_conf.get("freqai", {}).update({"fit_live_predictions_candles": 10})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
@ -418,12 +437,12 @@ def test_backtesting_fit_live_predictions(mocker, freqai_conf, caplog):
def test_plot_feature_importance(mocker, freqai_conf):
from freqtrade.freqai.utils import plot_feature_importance
freqai_conf.update({"timerange": "20180110-20180130"})
freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(
{"princpial_component_analysis": "true"})
{"princpial_component_analysis": "true"}
)
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
@ -436,15 +455,22 @@ def test_plot_feature_importance(mocker, freqai_conf):
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
freqai.dd.pair_dict = {"ADA/BTC": {"model_filename": "fake_name",
"trained_timestamp": 1, "data_path": "", "extras": {}}}
freqai.dd.pair_dict = {
"ADA/BTC": {
"model_filename": "fake_name",
"trained_timestamp": 1,
"data_path": "",
"extras": {},
}
}
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
new_timerange = TimeRange.parse_timerange("20180120-20180130")
freqai.dk.set_paths('ADA/BTC', None)
freqai.dk.set_paths("ADA/BTC", None)
freqai.extract_data_and_train_model(
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange
)
model = freqai.dd.load_data("ADA/BTC", freqai.dk)
@ -455,17 +481,21 @@ def test_plot_feature_importance(mocker, freqai_conf):
shutil.rmtree(Path(freqai.dk.full_path))
@pytest.mark.parametrize('timeframes,corr_pairs', [
(['5m'], ['ADA/BTC', 'DASH/BTC']),
(['5m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']),
(['5m', '15m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']),
])
@pytest.mark.parametrize(
"timeframes,corr_pairs",
[
(["5m"], ["ADA/BTC", "DASH/BTC"]),
(["5m"], ["ADA/BTC", "DASH/BTC", "ETH/USDT"]),
(["5m", "15m"], ["ADA/BTC", "DASH/BTC", "ETH/USDT"]),
],
)
def test_freqai_informative_pairs(mocker, freqai_conf, timeframes, corr_pairs):
freqai_conf['freqai']['feature_parameters'].update({
'include_timeframes': timeframes,
'include_corr_pairlist': corr_pairs,
})
freqai_conf["freqai"]["feature_parameters"].update(
{
"include_timeframes": timeframes,
"include_corr_pairlist": corr_pairs,
}
)
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
pairlists = PairListManager(exchange, freqai_conf)
@ -507,8 +537,8 @@ def test_download_all_data_for_training(mocker, freqai_conf, caplog, tmp_path):
exchange = get_patched_exchange(mocker, freqai_conf)
pairlist = PairListManager(exchange, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange, pairlist)
freqai_conf['pairs'] = freqai_conf['exchange']['pair_whitelist']
freqai_conf['datadir'] = tmp_path
freqai_conf["pairs"] = freqai_conf["exchange"]["pair_whitelist"]
freqai_conf["datadir"] = tmp_path
download_all_data_for_training(strategy.dp, freqai_conf)
assert log_has_re(
@ -518,9 +548,8 @@ def test_download_all_data_for_training(mocker, freqai_conf, caplog, tmp_path):
@pytest.mark.usefixtures("init_persistence")
@pytest.mark.parametrize('dp_exists', [(False), (True)])
@pytest.mark.parametrize("dp_exists", [(False), (True)])
def test_get_state_info(mocker, freqai_conf, dp_exists, caplog, tickers):
if is_mac():
pytest.skip("Reinforcement learning module not available on intel based Mac OS")
@ -528,12 +557,12 @@ def test_get_state_info(mocker, freqai_conf, dp_exists, caplog, tickers):
freqai_conf.update({"timerange": "20180110-20180130"})
freqai_conf.update({"strategy": "freqai_rl_test_strat"})
freqai_conf = make_rl_config(freqai_conf)
freqai_conf['entry_pricing']['price_side'] = 'same'
freqai_conf['exit_pricing']['price_side'] = 'same'
freqai_conf["entry_pricing"]["price_side"] = "same"
freqai_conf["exit_pricing"]["price_side"] = "same"
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
ticker_mock = MagicMock(return_value=tickers()['ETH/BTC'])
ticker_mock = MagicMock(return_value=tickers()["ETH/BTC"])
mocker.patch(f"{EXMS}.fetch_ticker", ticker_mock)
strategy.dp = DataProvider(freqai_conf, exchange)