Merge pull request #9179 from freqtrade/chore/better-freqai-reload

chore: fix bug associated with leaving FreqAI offline for more than 1candle
This commit is contained in:
Matthias 2023-09-14 18:10:47 +02:00 committed by GitHub
commit 6d8bf75572
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 158 additions and 14 deletions

View File

@ -263,23 +263,55 @@ class FreqaiDataDrawer:
self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy() self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy()
return return
def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None: def set_initial_return_values(self, pair: str,
pred_df: DataFrame,
dataframe: DataFrame
) -> None:
""" """
Set the initial return values to the historical predictions dataframe. This avoids needing Set the initial return values to the historical predictions dataframe. This avoids needing
to repredict on historical candles, and also stores historical predictions despite to repredict on historical candles, and also stores historical predictions despite
retrainings (so stored predictions are true predictions, not just inferencing on trained retrainings (so stored predictions are true predictions, not just inferencing on trained
data) data).
We also aim to keep the date from historical predictions so that the FreqUI displays
zeros during any downtime (between FreqAI reloads).
""" """
hist_df = self.historic_predictions new_pred = pred_df.copy()
len_diff = len(hist_df[pair].index) - len(pred_df.index) # set new_pred values to nans (we want to signal to user that there was nothing
if len_diff < 0: # historically made during downtime. The newest pred will get appeneded later in
df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]], # append_model_predictions)
ignore_index=True, keys=hist_df[pair].keys()) new_pred.iloc[:, :] = np.nan
new_pred["date"] = dataframe["date"]
hist_preds = self.historic_predictions[pair].copy()
# rename date_pred column to date so that we can merge on date
hist_preds = hist_preds.rename(columns={"date_pred": "date"})
# find the closest common date between new_pred and historic predictions
# and cut off the new_pred dataframe at that date
common_dates = pd.merge(new_pred, hist_preds, on="date", how="inner")
if len(common_dates.index) > 0:
new_pred = new_pred.iloc[len(common_dates):]
else: else:
df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True) logger.warning("No common dates found between new predictions and historic "
"predictions. You likely left your FreqAI instance offline "
f"for more than {len(dataframe.index)} candles.")
df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys())
# remove last row because we will append that later in append_model_predictions()
df_concat = df_concat.iloc[:-1]
# any missing values will get zeroed out so users can see the exact
# downtime in FreqUI
df_concat = df_concat.fillna(0) df_concat = df_concat.fillna(0)
self.model_return_values[pair] = df_concat
# rename date column back to date_pred
df_concat = df_concat.rename(columns={"date": "date_pred"})
self.historic_predictions[pair] = df_concat
self.model_return_values[pair] = df_concat.tail(len(dataframe.index)).reset_index(drop=True)
def append_model_predictions(self, pair: str, predictions: DataFrame, def append_model_predictions(self, pair: str, predictions: DataFrame,
do_preds: NDArray[np.int_], do_preds: NDArray[np.int_],

View File

@ -453,7 +453,7 @@ class IFreqaiModel(ABC):
pred_df, do_preds = self.predict(dataframe, dk) pred_df, do_preds = self.predict(dataframe, dk)
if pair not in self.dd.historic_predictions: if pair not in self.dd.historic_predictions:
self.set_initial_historic_predictions(pred_df, dk, pair, dataframe) self.set_initial_historic_predictions(pred_df, dk, pair, dataframe)
self.dd.set_initial_return_values(pair, pred_df) self.dd.set_initial_return_values(pair, pred_df, dataframe)
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe) dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
return return
@ -645,11 +645,11 @@ class IFreqaiModel(ABC):
If the user reuses an identifier on a subsequent instance, If the user reuses an identifier on a subsequent instance,
this function will not be called. In that case, "real" predictions this function will not be called. In that case, "real" predictions
will be appended to the loaded set of historic predictions. will be appended to the loaded set of historic predictions.
:param df: DataFrame = the dataframe containing the training feature data :param pred_df: DataFrame = the dataframe containing the predictions coming
:param model: Any = A model which was `fit` using a common library such as out of a model
catboost or lightgbm
:param dk: FreqaiDataKitchen = object containing methods for data analysis :param dk: FreqaiDataKitchen = object containing methods for data analysis
:param pair: str = current pair :param pair: str = current pair
:param strat_df: DataFrame = dataframe coming from strategy
""" """
self.dd.historic_predictions[pair] = pred_df self.dd.historic_predictions[pair] = pred_df

View File

@ -31,7 +31,7 @@ class FreqaiExampleStrategy(IStrategy):
plot_config = { plot_config = {
"main_plot": {}, "main_plot": {},
"subplots": { "subplots": {
"&-s_close": {"prediction": {"color": "blue"}}, "&-s_close": {"&-s_close": {"color": "blue"}},
"do_predict": { "do_predict": {
"do_predict": {"color": "brown"}, "do_predict": {"color": "brown"},
}, },

View File

@ -1,7 +1,9 @@
import shutil import shutil
from pathlib import Path from pathlib import Path
from unittest.mock import patch
import pandas as pd
import pytest import pytest
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
@ -135,3 +137,113 @@ def test_get_timerange_from_backtesting_live_df_pred_not_found(mocker, freqai_co
match=r'Historic predictions not found.*' match=r'Historic predictions not found.*'
): ):
freqai.dd.get_timerange_from_live_historic_predictions() freqai.dd.get_timerange_from_live_historic_predictions()
def test_set_initial_return_values(mocker, freqai_conf):
"""
Simple test of the set initial return values that ensures
we are concatening and ffilling values properly.
"""
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
freqai = strategy.freqai
freqai.live = False
freqai.dk = FreqaiDataKitchen(freqai_conf)
# Setup
pair = "BTC/USD"
end_x = "2023-08-31"
start_x_plus_1 = "2023-08-30"
end_x_plus_5 = "2023-09-03"
historic_data = {
'date_pred': pd.date_range(end=end_x, periods=5),
'value': range(1, 6)
}
new_data = {
'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5),
'value': range(6, 11)
}
freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data)
new_pred_df = pd.DataFrame(new_data)
dataframe = pd.DataFrame(new_data)
# Action
with patch('logging.Logger.warning') as mock_logger_warning:
freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe)
# Assertions
hist_pred_df = freqai.dd.historic_predictions[pair]
model_return_df = freqai.dd.model_return_values[pair]
assert (hist_pred_df['date_pred'].iloc[-1] ==
pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1))
assert 'date' not in hist_pred_df.columns
assert 'date_pred' in hist_pred_df.columns
assert hist_pred_df.shape[0] == 7 # Total rows: 5 from historic and 2 new zeros
# compare values in model_return_df with hist_pred_df
assert (model_return_df["value"].values ==
hist_pred_df.tail(len(dataframe))["value"].values).all()
assert model_return_df.shape[0] == len(dataframe)
# Ensure logger error is not called
mock_logger_warning.assert_not_called()
def test_set_initial_return_values_warning(mocker, freqai_conf):
"""
Simple test of set_initial_return_values that hits the warning
associated with leaving a FreqAI bot offline so long that the
exchange candles have no common date with the historic predictions
"""
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
freqai = strategy.freqai
freqai.live = False
freqai.dk = FreqaiDataKitchen(freqai_conf)
# Setup
pair = "BTC/USD"
end_x = "2023-08-31"
start_x_plus_1 = "2023-09-01"
end_x_plus_5 = "2023-09-05"
historic_data = {
'date_pred': pd.date_range(end=end_x, periods=5),
'value': range(1, 6)
}
new_data = {
'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5),
'value': range(6, 11)
}
freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data)
new_pred_df = pd.DataFrame(new_data)
dataframe = pd.DataFrame(new_data)
# Action
with patch('logging.Logger.warning') as mock_logger_warning:
freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe)
# Assertions
hist_pred_df = freqai.dd.historic_predictions[pair]
model_return_df = freqai.dd.model_return_values[pair]
assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1)
assert 'date' not in hist_pred_df.columns
assert 'date_pred' in hist_pred_df.columns
assert hist_pred_df.shape[0] == 9 # Total rows: 5 from historic and 4 new zeros
# compare values in model_return_df with hist_pred_df
assert (model_return_df["value"].values == hist_pred_df.tail(
len(dataframe))["value"].values).all()
assert model_return_df.shape[0] == len(dataframe)
# Ensure logger error is not called
mock_logger_warning.assert_called()