From 628963c20704ca7d47f4d2ec79cfaaf048fc8c82 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 12 Sep 2023 12:19:12 +0200 Subject: [PATCH 1/3] chore: fix bug associated with leaving FreqAI offline for more than 1 candle. --- freqtrade/freqai/data_drawer.py | 50 ++++++++++++++++---- freqtrade/freqai/freqai_interface.py | 2 +- freqtrade/templates/FreqaiExampleStrategy.py | 2 +- 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index b6ded83b1..49c673adc 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -263,23 +263,55 @@ class FreqaiDataDrawer: self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy() return - def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None: + def set_initial_return_values(self, pair: str, + pred_df: DataFrame, + dataframe: DataFrame + ) -> None: """ Set the initial return values to the historical predictions dataframe. This avoids needing to repredict on historical candles, and also stores historical predictions despite retrainings (so stored predictions are true predictions, not just inferencing on trained - data) + data). + + We also aim to keep the date from historical predictions so that the FreqUI displays + zeros during any downtime (between FreqAI reloads). """ - hist_df = self.historic_predictions - len_diff = len(hist_df[pair].index) - len(pred_df.index) - if len_diff < 0: - df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]], - ignore_index=True, keys=hist_df[pair].keys()) + new_pred = pred_df.copy() + # set new_pred values to nans (we want to signal to user that there was nothing + # historically made during downtime. The newest pred will get appeneded later in + # append_model_predictions) + new_pred.iloc[:, :] = np.nan + new_pred["date"] = dataframe["date"] + + hist_preds = self.historic_predictions[pair].copy() + # rename date_pred column to date so that we can merge on date + hist_preds = hist_preds.rename(columns={"date_pred": "date"}) + + # find the closest common date between new_pred and historic predictions + # and cut off the new_pred dataframe at that date + common_dates = pd.merge(new_pred, hist_preds, on="date", how="inner") + if len(common_dates.index) > 0: + new_pred = new_pred.iloc[len(common_dates):] else: - df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True) + logger.error("No common dates found between new predictions and historic predictions. " + "You likely left your FreqAI instance offline for more than " + f"{len(dataframe.index)} candles.") + + df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys()) + + # remove last row because we will append that later in append_model_predictions() + df_concat = df_concat.iloc[:-1] + # any missing values will get zeroed out so users can see the exact + # downtime in FreqUI df_concat = df_concat.fillna(0) - self.model_return_values[pair] = df_concat + + # rename date column back to date_pred + df_concat = df_concat.rename(columns={"date": "date_pred"}) + + self.historic_predictions[pair] = df_concat + + self.model_return_values[pair] = df_concat.tail(len(dataframe.index)).reset_index(drop=True) def append_model_predictions(self, pair: str, predictions: DataFrame, do_preds: NDArray[np.int_], diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index efae6d060..f64a3b8f0 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -453,7 +453,7 @@ class IFreqaiModel(ABC): pred_df, do_preds = self.predict(dataframe, dk) if pair not in self.dd.historic_predictions: self.set_initial_historic_predictions(pred_df, dk, pair, dataframe) - self.dd.set_initial_return_values(pair, pred_df) + self.dd.set_initial_return_values(pair, pred_df, dataframe) dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe) return diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 084cf2e89..e64570b9e 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -31,7 +31,7 @@ class FreqaiExampleStrategy(IStrategy): plot_config = { "main_plot": {}, "subplots": { - "&-s_close": {"prediction": {"color": "blue"}}, + "&-s_close": {"&-s_close": {"color": "blue"}}, "do_predict": { "do_predict": {"color": "brown"}, }, From 844ab4aef5d5f7dc986e9f83d517df9883695625 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 14 Sep 2023 00:05:59 +0200 Subject: [PATCH 2/3] chore: add tests for set_initial_return_values --- freqtrade/freqai/data_drawer.py | 6 +- freqtrade/freqai/freqai_interface.py | 6 +- tests/freqai/test_freqai_datadrawer.py | 120 +++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 49c673adc..013300dfe 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -294,9 +294,9 @@ class FreqaiDataDrawer: if len(common_dates.index) > 0: new_pred = new_pred.iloc[len(common_dates):] else: - logger.error("No common dates found between new predictions and historic predictions. " - "You likely left your FreqAI instance offline for more than " - f"{len(dataframe.index)} candles.") + logger.warning("No common dates found between new predictions and historic " + "predictions. You likely left your FreqAI instance offline " + f"for more than {len(dataframe.index)} candles.") df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys()) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index f64a3b8f0..33d23aa73 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -645,11 +645,11 @@ class IFreqaiModel(ABC): If the user reuses an identifier on a subsequent instance, this function will not be called. In that case, "real" predictions will be appended to the loaded set of historic predictions. - :param df: DataFrame = the dataframe containing the training feature data - :param model: Any = A model which was `fit` using a common library such as - catboost or lightgbm + :param pred_df: DataFrame = the dataframe containing the predictions coming + out of a model :param dk: FreqaiDataKitchen = object containing methods for data analysis :param pair: str = current pair + :param strat_df: DataFrame = dataframe coming from strategy """ self.dd.historic_predictions[pair] = pred_df diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index 8ab2c75da..b7414884f 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -1,7 +1,9 @@ import shutil from pathlib import Path +from unittest.mock import patch +import pandas as pd import pytest from freqtrade.configuration import TimeRange @@ -135,3 +137,121 @@ def test_get_timerange_from_backtesting_live_df_pred_not_found(mocker, freqai_co match=r'Historic predictions not found.*' ): freqai.dd.get_timerange_from_live_historic_predictions() + + +class MockClass: # This represents your class that has the set_initial_return_values method. + def __init__(self): + self.historic_predictions = {} + self.model_return_values = {} + + # ... set_initial_return_values function here ... + + +def test_set_initial_return_values(mocker, freqai_conf): + """ + Simple test of the set initial return values that ensures + we are concatening and ffilling values properly. + """ + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + # Setup + pair = "BTC/USD" + end_x = "2023-08-31" + start_x_plus_1 = "2023-08-30" + end_x_plus_5 = "2023-09-03" + + historic_data = { + 'date_pred': pd.date_range(end=end_x, periods=5), + 'value': range(1, 6) + } + new_data = { + 'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5), + 'value': range(6, 11) + } + + freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data) + + new_pred_df = pd.DataFrame(new_data) + dataframe = pd.DataFrame(new_data) + + # Action + with patch('logging.Logger.warning') as mock_logger_warning: + freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe) + + # Assertions + hist_pred_df = freqai.dd.historic_predictions[pair] + model_return_df = freqai.dd.model_return_values[pair] + + assert (hist_pred_df['date_pred'].iloc[-1] == + pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1)) + assert 'date' not in hist_pred_df.columns + assert 'date_pred' in hist_pred_df.columns + assert hist_pred_df.shape[0] == 7 # Total rows: 5 from historic and 2 new zeros + + # compare values in model_return_df with hist_pred_df + assert (model_return_df["value"].values == + hist_pred_df.tail(len(dataframe))["value"].values).all() + assert model_return_df.shape[0] == len(dataframe) + + # Ensure logger error is not called + mock_logger_warning.assert_not_called() + + +def test_set_initial_return_values_warning(mocker, freqai_conf): + """ + Simple test of set_initial_return_values that hits the warning + associated with leaving a FreqAI bot offline so long that the + exchange candles have no common date with the historic predictions + """ + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + # Setup + pair = "BTC/USD" + end_x = "2023-08-31" + start_x_plus_1 = "2023-09-01" + end_x_plus_5 = "2023-09-05" + + historic_data = { + 'date_pred': pd.date_range(end=end_x, periods=5), + 'value': range(1, 6) + } + new_data = { + 'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5), + 'value': range(6, 11) + } + + freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data) + + new_pred_df = pd.DataFrame(new_data) + dataframe = pd.DataFrame(new_data) + + # Action + with patch('logging.Logger.warning') as mock_logger_warning: + freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe) + + # Assertions + hist_pred_df = freqai.dd.historic_predictions[pair] + model_return_df = freqai.dd.model_return_values[pair] + + assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1) + assert 'date' not in hist_pred_df.columns + assert 'date_pred' in hist_pred_df.columns + assert hist_pred_df.shape[0] == 9 # Total rows: 5 from historic and 4 new zeros + + # compare values in model_return_df with hist_pred_df + assert (model_return_df["value"].values == hist_pred_df.tail( + len(dataframe))["value"].values).all() + assert model_return_df.shape[0] == len(dataframe) + + # Ensure logger error is not called + mock_logger_warning.assert_called() From 310c9f60341ccf017af629e07c2bb50a4e3faabc Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Thu, 14 Sep 2023 13:49:31 +0200 Subject: [PATCH 3/3] Update test_freqai_datadrawer.py --- tests/freqai/test_freqai_datadrawer.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index b7414884f..b511f3c7b 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -139,14 +139,6 @@ def test_get_timerange_from_backtesting_live_df_pred_not_found(mocker, freqai_co freqai.dd.get_timerange_from_live_historic_predictions() -class MockClass: # This represents your class that has the set_initial_return_values method. - def __init__(self): - self.historic_predictions = {} - self.model_return_values = {} - - # ... set_initial_return_values function here ... - - def test_set_initial_return_values(mocker, freqai_conf): """ Simple test of the set initial return values that ensures