only warn on exceptions in backtest for freqai interface

This commit is contained in:
Aleksey Savin 2024-04-18 09:09:01 +00:00
parent eddca5a694
commit c537c43dd0
2 changed files with 35 additions and 15 deletions

View File

@ -48,6 +48,7 @@
],
"freqai": {
"enabled": true,
"warn_exceptions_on_backtest_only": false,
"purge_old_models": 2,
"train_period_days": 15,
"backtest_period_days": 7,

View File

@ -116,6 +116,8 @@ class IFreqaiModel(ABC):
record_params(config, self.full_path)
self.new_feature_selector = self.config.get('freqai', {}).get('warn_exceptions_on_backtest_only', False)
def __getstate__(self):
"""
Return an empty state to be pickled in hyperopt
@ -349,30 +351,47 @@ class IFreqaiModel(ABC):
self.tb_logger = get_tb_logger(self.dd.model_type, dk.data_path,
self.activate_tensorboard)
self.model = self.train(dataframe_train, pair, dk)
self.tb_logger.close()
self.tb_logger.close()
except Exception as msg:
logger.warning(
f"Training {pair} raised exception {msg.__class__.__name__}. "
f"Message: {msg}, skipping.", exc_info=True)
if self.new_feature_selector:
logger.warning(
"Train failed. Try to train on next pair." if self.new_feature_selector else
"Train failed. Raise error, fix data issue and try again."
)
self.tb_logger.close()
self.model = None
self.dd.pair_dict[pair]["trained_timestamp"] = int(
tr_train.stopts)
if self.plot_features and self.model is not None:
plot_feature_importance(self.model, pair, dk, self.plot_features)
if self.save_backtest_models and self.model is not None:
logger.info('Saving backtest model to disk.')
self.dd.save_data(self.model, pair, dk)
else:
logger.info('Saving metadata to disk.')
self.dd.save_metadata(dk)
hard_check_model_valid = True
if self.new_feature_selector:
hard_check_model_valid = bool(False if self.model is None else True)
if hard_check_model_valid:
logger.info(
"Model is trained. Saving model and metadata to disk."
)
if hard_check_model_valid:
self.dd.pair_dict[pair]["trained_timestamp"] = int(tr_train.stopts)
if self.plot_features and self.model is not None:
plot_feature_importance(self.model, pair, dk, self.plot_features)
if self.save_backtest_models and self.model is not None:
logger.info('Saving backtest model to disk.')
self.dd.save_data(self.model, pair, dk)
else:
logger.info('Saving metadata to disk.')
self.dd.save_metadata(dk)
else:
self.model = self.dd.load_data(pair, dk)
pred_df, do_preds = self.predict(dataframe_backtest, dk)
append_df = dk.get_predictions_to_append(pred_df, do_preds, dataframe_backtest)
dk.append_predictions(append_df)
dk.save_backtesting_prediction(append_df)
if hard_check_model_valid:
pred_df, do_preds = self.predict(dataframe_backtest, dk)
append_df = dk.get_predictions_to_append(pred_df, do_preds, dataframe_backtest)
dk.append_predictions(append_df)
dk.save_backtesting_prediction(append_df)
self.backtesting_fit_live_predictions(dk)
dk.fill_predictions(dataframe)