From ce8fbbf7433ee1f1542495b55d6d821e23d20549 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Sat, 6 Aug 2022 07:25:59 +0200 Subject: [PATCH] ensure loading historical df matches frequi indices --- freqtrade/freqai/data_drawer.py | 3 ++- freqtrade/freqai/data_kitchen.py | 19 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index f8022c8e0..85a84160f 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -238,9 +238,10 @@ class FreqaiDataDrawer: df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]], ignore_index=True, keys=hist_df[pair].keys()) else: - df_concat = hist_df[pair].tail(len(pred_df.index)) + df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True) df_concat = df_concat.fillna(0) self.model_return_values[pair] = df_concat + logger.info(f'Setting initial FreqUI plots from historical data for {pair}.') else: for label in dk.label_list: diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 52b251ddf..6fb75374d 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -4,7 +4,7 @@ import logging import shutil import sqlite3 from pathlib import Path -from typing import Any, Dict, List, Tuple, Optional +from typing import Any, Dict, List, Optional, Tuple import numpy as np import numpy.typing as npt @@ -14,6 +14,7 @@ from sklearn import linear_model from sklearn.cluster import DBSCAN from sklearn.metrics.pairwise import pairwise_distances from sklearn.model_selection import train_test_split +from sklearn.neighbors import NearestNeighbors from freqtrade.configuration import TimeRange from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data @@ -21,8 +22,6 @@ from freqtrade.exceptions import OperationalException from freqtrade.resolvers import ExchangeResolver from freqtrade.strategy.interface import IStrategy -from sklearn.neighbors import NearestNeighbors - SECONDS_IN_DAY = 86400 SECONDS_IN_HOUR = 3600 @@ -505,7 +504,7 @@ class FreqaiDataKitchen: point. This metric defines the neighborhood of trained data and is used for prediction confidence in the Dissimilarity Index """ - logger.info("computing average mean distance for all training points") + # logger.info("computing average mean distance for all training points") pairwise = pairwise_distances( self.data_dictionary["train_features"], n_jobs=self.thread_count) avg_mean_dist = pairwise.mean(axis=1).mean() @@ -538,7 +537,7 @@ class FreqaiDataKitchen: if (len(do_predict) - do_predict.sum()) > 0: logger.info( - f"svm_remove_outliers() tossed {len(do_predict) - do_predict.sum()} predictions" + f"SVM tossed {len(do_predict) - do_predict.sum()} predictions." ) self.do_predict += do_predict self.do_predict -= 1 @@ -564,8 +563,8 @@ class FreqaiDataKitchen: ] logger.info( - f"svm_remove_outliers() tossed {len(y_pred) - dropped_points.sum()}" - f" train points from {len(y_pred)}" + f"SVM tossed {len(y_pred) - dropped_points.sum()}" + f" train points from {len(y_pred)} total points." ) # same for test data @@ -582,8 +581,8 @@ class FreqaiDataKitchen: ] logger.info( - f"svm_remove_outliers() tossed {len(y_pred) - dropped_points.sum()}" - f" test points from {len(y_pred)}" + f"SVM tossed {len(y_pred) - dropped_points.sum()}" + f" test points from {len(y_pred)} total points." ) return @@ -621,7 +620,7 @@ class FreqaiDataKitchen: else: - MinPts = len(self.data_dictionary['train_features'].columns)*2 + MinPts = len(self.data_dictionary['train_features'].columns) * 2 # measure pairwise distances to train_features.shape[1]*2 nearest neighbours neighbors = NearestNeighbors( n_neighbors=MinPts, n_jobs=self.thread_count)