From 2403a03fcbbae135b9919e635ef8d63257610c97 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Aug 2022 06:28:53 +0200 Subject: [PATCH 01/27] Version bump 2022.8 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 2572c03f1..6c5c52a04 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.8.dev' +__version__ = '2022.8' if 'dev' in __version__: try: From d0456b698cb2ed8f60d8a3b80165c36b657bf59c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Sep 2022 07:22:41 +0200 Subject: [PATCH 02/27] Version bump 2022.9 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 6c5c52a04..634377e05 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.8' +__version__ = '2022.9' if 'dev' in __version__: try: From 8e101a9f1c3d0b76ef7be383644f62b2cf178d41 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Oct 2022 09:35:21 +0200 Subject: [PATCH 03/27] Disable log spam from analyze_df in webhook/discord --- freqtrade/rpc/discord.py | 2 +- freqtrade/rpc/webhook.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/discord.py b/freqtrade/rpc/discord.py index 9efe6f427..c48508300 100644 --- a/freqtrade/rpc/discord.py +++ b/freqtrade/rpc/discord.py @@ -30,9 +30,9 @@ class Discord(Webhook): pass def send_msg(self, msg) -> None: - logger.info(f"Sending discord message: {msg}") if msg['type'].value in self.config['discord']: + logger.info(f"Sending discord message: {msg}") msg['strategy'] = self.strategy msg['timeframe'] = self.timeframe diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 6109e80bc..bb3b3922f 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -61,6 +61,14 @@ class Webhook(RPCHandler): RPCMessageType.STARTUP, RPCMessageType.WARNING): valuedict = whconfig.get('webhookstatus') + elif msg['type'] in ( + RPCMessageType.PROTECTION_TRIGGER, + RPCMessageType.PROTECTION_TRIGGER_GLOBAL, + RPCMessageType.WHITELIST, + RPCMessageType.ANALYZED_DF, + RPCMessageType.STRATEGY_MSG): + # Don't fail for non-implemented types + return else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) if not valuedict: From 6841bdaa8193278d6863131f8866f4a59286da37 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Oct 2022 09:45:58 +0200 Subject: [PATCH 04/27] Update test to verify webhook won't log-spam on new messagetypes --- tests/rpc/test_rpc_webhook.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 4d65b4966..3bbb85d54 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -365,6 +365,14 @@ def test_exception_send_msg(default_conf, mocker, caplog): with pytest.raises(NotImplementedError): webhook.send_msg(msg) + # Test no failure for not implemented but known messagetypes + for e in RPCMessageType: + msg = { + 'type': e, + 'status': 'whatever' + } + webhook.send_msg(msg) + def test__send_msg(default_conf, mocker, caplog): default_conf["webhook"] = get_webhook_dict() From 19b3669d971231654c3ff0ff5a52a6fe6a4e447e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Oct 2022 21:20:14 +0200 Subject: [PATCH 05/27] Decrease message throughput fixes memory leak by queue raising indefinitely --- freqtrade/rpc/api_server/webserver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index df4324740..53af91477 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -198,8 +198,10 @@ class ApiServer(RPCHandler): logger.debug(f"Found message of type: {message.get('type')}") # Broadcast it await self._ws_channel_manager.broadcast(message) - # Sleep, make this configurable? - await asyncio.sleep(0.1) + # Limit messages per sec. + # Could cause problems with queue size if too low, and + # problems with network traffik if too high. + await asyncio.sleep(0.001) except asyncio.CancelledError: pass From 03256fc7768bfdebaa8ecbb9e547682ab36f0591 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Sat, 1 Oct 2022 16:50:29 +0200 Subject: [PATCH 06/27] Merge pull request #7508 from aemr3/fix-pca-errors Fix feature list match for PCA --- .../freqai/base_models/BaseClassifierModel.py | 2 +- .../freqai/base_models/BaseRegressionModel.py | 2 +- freqtrade/freqai/data_drawer.py | 2 +- freqtrade/freqai/data_kitchen.py | 1 + freqtrade/freqai/freqai_interface.py | 17 +++++++++-------- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqai/base_models/BaseClassifierModel.py b/freqtrade/freqai/base_models/BaseClassifierModel.py index 70f212d2a..09f1bf98c 100644 --- a/freqtrade/freqai/base_models/BaseClassifierModel.py +++ b/freqtrade/freqai/base_models/BaseClassifierModel.py @@ -92,7 +92,7 @@ class BaseClassifierModel(IFreqaiModel): filtered_df = dk.normalize_data_from_metadata(filtered_df) dk.data_dictionary["prediction_features"] = filtered_df - self.data_cleaning_predict(dk, filtered_df) + self.data_cleaning_predict(dk) predictions = self.model.predict(dk.data_dictionary["prediction_features"]) pred_df = DataFrame(predictions, columns=dk.label_list) diff --git a/freqtrade/freqai/base_models/BaseRegressionModel.py b/freqtrade/freqai/base_models/BaseRegressionModel.py index 2450bf305..5d89dd356 100644 --- a/freqtrade/freqai/base_models/BaseRegressionModel.py +++ b/freqtrade/freqai/base_models/BaseRegressionModel.py @@ -92,7 +92,7 @@ class BaseRegressionModel(IFreqaiModel): dk.data_dictionary["prediction_features"] = filtered_df # optional additional data cleaning/analysis - self.data_cleaning_predict(dk, filtered_df) + self.data_cleaning_predict(dk) predictions = self.model.predict(dk.data_dictionary["prediction_features"]) pred_df = DataFrame(predictions, columns=dk.label_list) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 1839724f8..471f6875c 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -423,7 +423,7 @@ class FreqaiDataDrawer: dk.data["data_path"] = str(dk.data_path) dk.data["model_filename"] = str(dk.model_filename) - dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns) + dk.data["training_features_list"] = dk.training_features_list dk.data["label_list"] = dk.label_list # store the metadata with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp: diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index f4fa4e5fd..697fd85cf 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -881,6 +881,7 @@ class FreqaiDataKitchen: """ column_names = dataframe.columns features = [c for c in column_names if "%" in c] + if not features: raise OperationalException("Could not find any features!") diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index d9f917338..78539bae5 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -275,7 +275,8 @@ class IFreqaiModel(ABC): if dk.check_if_backtest_prediction_exists(): self.dd.load_metadata(dk) - self.check_if_feature_list_matches_strategy(dataframe_train, dk) + dk.find_features(dataframe_train) + self.check_if_feature_list_matches_strategy(dk) append_df = dk.get_backtesting_prediction() dk.append_predictions(append_df) else: @@ -296,7 +297,6 @@ class IFreqaiModel(ABC): else: self.model = self.dd.load_data(pair, dk) - # self.check_if_feature_list_matches_strategy(dataframe_train, dk) pred_df, do_preds = self.predict(dataframe_backtest, dk) append_df = dk.get_predictions_to_append(pred_df, do_preds) dk.append_predictions(append_df) @@ -420,7 +420,7 @@ class IFreqaiModel(ABC): return def check_if_feature_list_matches_strategy( - self, dataframe: DataFrame, dk: FreqaiDataKitchen + self, dk: FreqaiDataKitchen ) -> None: """ Ensure user is passing the proper feature set if they are reusing an `identifier` pointing @@ -429,11 +429,12 @@ class IFreqaiModel(ABC): :param dk: FreqaiDataKitchen = non-persistent data container/analyzer for current coin/bot loop """ - dk.find_features(dataframe) + if "training_features_list_raw" in dk.data: feature_list = dk.data["training_features_list_raw"] else: feature_list = dk.data['training_features_list'] + if dk.training_features_list != feature_list: raise OperationalException( "Trying to access pretrained model with `identifier` " @@ -481,13 +482,16 @@ class IFreqaiModel(ABC): if self.freqai_info["feature_parameters"].get('noise_standard_deviation', 0): dk.add_noise_to_training_features() - def data_cleaning_predict(self, dk: FreqaiDataKitchen, dataframe: DataFrame) -> None: + def data_cleaning_predict(self, dk: FreqaiDataKitchen) -> None: """ Base data cleaning method for predict. Functions here are complementary to the functions of data_cleaning_train. """ ft_params = self.freqai_info["feature_parameters"] + # ensure user is feeding the correct indicators to the model + self.check_if_feature_list_matches_strategy(dk) + if ft_params.get('inlier_metric_window', 0): dk.compute_inlier_metric(set_='predict') @@ -505,9 +509,6 @@ class IFreqaiModel(ABC): if ft_params.get("use_DBSCAN_to_remove_outliers", False): dk.use_DBSCAN_to_remove_outliers(predict=True) - # ensure user is feeding the correct indicators to the model - self.check_if_feature_list_matches_strategy(dk.data_dictionary['prediction_features'], dk) - def model_exists(self, dk: FreqaiDataKitchen) -> bool: """ Given a pair and path, check if a model already exists From c53ff94b8e5de6a2cbbad69f386eb1c717211ce2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Sep 2022 13:47:26 +0200 Subject: [PATCH 07/27] Force joblib update via setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0581081fa..d3f9ea7c0 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setup( 'pandas', 'tables', 'blosc', - 'joblib', + 'joblib>=1.2.0', 'pyarrow; platform_machine != "armv7l"', 'fastapi', 'uvicorn', From 59cfde3767f449c588500acb07c12503ac3d9032 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Sep 2022 15:43:05 +0200 Subject: [PATCH 08/27] Fix pandas deprecation warnings from freqAI --- freqtrade/freqai/data_kitchen.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 697fd85cf..766eb981f 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -210,7 +210,7 @@ class FreqaiDataKitchen: filtered_df = unfiltered_df.filter(training_feature_list, axis=1) filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan) - drop_index = pd.isnull(filtered_df).any(1) # get the rows that have NaNs, + drop_index = pd.isnull(filtered_df).any(axis=1) # get the rows that have NaNs, drop_index = drop_index.replace(True, 1).replace(False, 0) # pep8 requirement. if (training_filter): const_cols = list((filtered_df.nunique() == 1).loc[lambda x: x].index) @@ -221,7 +221,7 @@ class FreqaiDataKitchen: # about removing any row with NaNs # if labels has multiple columns (user wants to train multiple modelEs), we detect here labels = unfiltered_df.filter(label_list, axis=1) - drop_index_labels = pd.isnull(labels).any(1) + drop_index_labels = pd.isnull(labels).any(axis=1) drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0) dates = unfiltered_df['date'] filtered_df = filtered_df[ @@ -249,7 +249,7 @@ class FreqaiDataKitchen: else: # we are backtesting so we need to preserve row number to send back to strategy, # so now we use do_predict to avoid any prediction based on a NaN - drop_index = pd.isnull(filtered_df).any(1) + drop_index = pd.isnull(filtered_df).any(axis=1) self.data["filter_drop_index_prediction"] = drop_index filtered_df.fillna(0, inplace=True) # replacing all NaNs with zeros to avoid issues in 'prediction', but any prediction @@ -808,7 +808,7 @@ class FreqaiDataKitchen: :, :no_prev_pts ] distances = distances.replace([np.inf, -np.inf], np.nan) - drop_index = pd.isnull(distances).any(1) + drop_index = pd.isnull(distances).any(axis=1) distances = distances[drop_index == 0] inliers = pd.DataFrame(index=distances.index) From 851d1e9da106aad2153ec38919873f61ce4b384f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Oct 2022 06:56:02 +0200 Subject: [PATCH 09/27] Version bump 2022.9.1 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 634377e05..2bd09a377 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.9' +__version__ = '2022.9.1' if 'dev' in __version__: try: From ec7d6634963cd9482f47e5db0c6e90e9ffd176ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 Oct 2022 19:34:30 +0200 Subject: [PATCH 10/27] Version bump 2022.10 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index ce5f420e4..74c558a37 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.9.1' +__version__ = '2022.10' if 'dev' in __version__: try: From 5c571f565ff639d10b5095db7f7429e35a9c0b73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Nov 2022 15:34:00 +0100 Subject: [PATCH 11/27] Version bump 2022.11 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 74c558a37..47d095f65 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.10' +__version__ = '2022.11' if 'dev' in __version__: try: From 8e8f71ade560efe5364c6bc3238168ecd9631e04 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 28 Dec 2022 15:42:38 +0100 Subject: [PATCH 12/27] Version bump 2022.12 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index b44189cb0..4c0f570b1 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.12.dev' +__version__ = '2022.12' if 'dev' in __version__: try: From 786f7469586ae4705446bb8e6233fb22e3d32dd4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Jan 2023 07:11:02 +0100 Subject: [PATCH 13/27] Version bump to 2023.1 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 18b6c9130..f531bb605 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.1.dev' +__version__ = '2023.1' if 'dev' in __version__: from pathlib import Path From 75bc5809a9b18bf1c1b97ce922046fe764be62f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Feb 2023 20:02:51 +0100 Subject: [PATCH 14/27] Better handle backtest errors --- freqtrade/rpc/api_server/api_backtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index ce71467ca..77de33994 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -118,7 +118,7 @@ async def api_start_backtest( # noqa: C901 logger.info("Backtest finished.") - except (OperationalException, DependencyException) as e: + except (Exception, OperationalException, DependencyException) as e: logger.exception(f"Backtesting caused an error: {e}") pass finally: From e6766b9b82805d0673155b7da7fc8eca121ec21c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Feb 2023 20:22:59 +0100 Subject: [PATCH 15/27] Add bt-error to UI backtest method. --- freqtrade/rpc/api_server/api_backtest.py | 10 ++++++++++ freqtrade/rpc/api_server/webserver.py | 1 + tests/rpc/test_rpc_apiserver.py | 10 ++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 77de33994..d9d7a27f1 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -29,6 +29,7 @@ router = APIRouter() async def api_start_backtest( # noqa: C901 bt_settings: BacktestRequest, background_tasks: BackgroundTasks, config=Depends(get_config), ws_mode=Depends(is_webserver_mode)): + ApiServer._bt['bt_error'] = None """Start backtesting if not done so already""" if ApiServer._bgtask_running: raise RPCException('Bot Background task already running') @@ -120,6 +121,7 @@ async def api_start_backtest( # noqa: C901 except (Exception, OperationalException, DependencyException) as e: logger.exception(f"Backtesting caused an error: {e}") + ApiServer._bt['bt_error'] = str(e) pass finally: ApiServer._bgtask_running = False @@ -162,6 +164,14 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest not yet executed" } + if ApiServer._bt['bt_error']: + return { + "status": "error", + "running": False, + "step": "", + "progress": 0, + "status_msg": f"Backtest failed with {ApiServer._bt['bt_error']}" + } return { "status": "ended", diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index b3ef794d8..b53662451 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -41,6 +41,7 @@ class ApiServer(RPCHandler): 'data': None, 'timerange': None, 'last_config': {}, + 'bt_error': None, } _has_rpc: bool = False _bgtask_running: bool = False diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 94b210c76..43d9abb78 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1737,9 +1737,15 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): data['stake_amount'] = 101 mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', - side_effect=DependencyException()) + side_effect=DependencyException('DeadBeef')) rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert log_has("Backtesting caused an error: ", caplog) + assert log_has("Backtesting caused an error: DeadBeef", caplog) + + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'error' + assert 'Backtest failed' in result['status_msg'] # Delete backtesting to avoid leakage since the backtest-object may stick around. rc = client_delete(client, f"{BASE_URI}/backtest") From 2bc9413be11288d0b42c0c354cb2d988eca5af22 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Feb 2023 20:58:24 +0100 Subject: [PATCH 16/27] Fix minor stylistic errors --- freqtrade/__main__.py | 0 freqtrade/commands/analyze_commands.py | 0 freqtrade/commands/hyperopt_commands.py | 0 freqtrade/data/entryexitanalysis.py | 0 freqtrade/optimize/hyperopt_tools.py | 0 freqtrade/vendor/qtpylib/indicators.py | 1 - freqtrade/worker.py | 0 scripts/ws_client.py | 0 tests/data/test_entryexitanalysis.py | 0 tests/exchange/test_ccxt_compat.py | 2 +- tests/strategy/test_interface.py | 4 ++-- 11 files changed, 3 insertions(+), 4 deletions(-) mode change 100644 => 100755 freqtrade/__main__.py mode change 100755 => 100644 freqtrade/commands/analyze_commands.py mode change 100755 => 100644 freqtrade/commands/hyperopt_commands.py mode change 100755 => 100644 freqtrade/data/entryexitanalysis.py mode change 100755 => 100644 freqtrade/optimize/hyperopt_tools.py mode change 100755 => 100644 freqtrade/worker.py mode change 100644 => 100755 scripts/ws_client.py mode change 100755 => 100644 tests/data/test_entryexitanalysis.py diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py old mode 100644 new mode 100755 diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py old mode 100755 new mode 100644 diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py old mode 100755 new mode 100644 diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py old mode 100755 new mode 100644 diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py old mode 100755 new mode 100644 diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index 4f14ae13c..3da4f038d 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # QTPyLib: Quantitative Trading Python Library diff --git a/freqtrade/worker.py b/freqtrade/worker.py old mode 100755 new mode 100644 diff --git a/scripts/ws_client.py b/scripts/ws_client.py old mode 100644 new mode 100755 diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py old mode 100755 new mode 100644 diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index f1d240f9f..bbeb56c6a 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -309,7 +309,7 @@ def exchange(request, exchange_conf): @pytest.fixture(params=EXCHANGES, scope="class") def exchange_futures(request, exchange_conf, class_mocker): - if not EXCHANGES[request.param].get('futures') is True: + if EXCHANGES[request.param].get('futures') is not True: yield None, request.param else: exchange_conf = set_test_proxy( diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index fe562907a..0b30d2059 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -214,12 +214,12 @@ def test_ignore_expired_candle(default_conf): current_time = latest_date + timedelta(seconds=30 + 300) - assert not strategy.ignore_expired_candle( + assert strategy.ignore_expired_candle( latest_date=latest_date, current_time=current_time, timeframe_seconds=300, enter=True - ) is True + ) is not True def test_assert_df_raise(mocker, caplog, ohlcv_history): From 549a0e1c447df9fe75ee834dad58d83ee4cd8bb9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Feb 2023 21:06:07 +0100 Subject: [PATCH 17/27] Add ruff linting - initial configuration --- .github/workflows/ci.yml | 12 ++++++++++++ .pre-commit-config.yaml | 6 ++++++ pyproject.toml | 14 ++++++++++++++ requirements-dev.txt | 1 + 4 files changed, 33 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfc8ac3b6..4cff9ef7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,10 @@ jobs: run: | isort --check . + - name: Ruff - linting + run: | + ruff check . + - name: Mypy run: | mypy freqtrade scripts tests @@ -194,6 +198,10 @@ jobs: run: | isort --check . + - name: Ruff - linting + run: | + ruff check . + - name: Mypy run: | mypy freqtrade scripts @@ -252,6 +260,10 @@ jobs: run: | flake8 + - name: Ruff - linting + run: | + ruff check . + - name: Mypy run: | mypy freqtrade scripts tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57ce81b8c..58f526ce9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,12 @@ repos: name: isort (python) # stages: [push] + - repo: https://github.com/charliermarsh/ruff-pre-commit + # Ruff version. + rev: 'v0.0.251' + hooks: + - id: ruff + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 82d4ceaf8..7da2a2200 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,3 +56,17 @@ exclude = [ "build_helpers/*.py", ] ignore = ["freqtrade/vendor/**"] + + +[tool.ruff] +line-length = 100 +extend-exclude = [".env"] +extend-select = [ + "TID", + "EXE", + "YTT", + # "DTZ", + # "RSE", + # "TCH", + # "PTH", +] diff --git a/requirements-dev.txt b/requirements-dev.txt index 32b7cfcc5..287cb8ae9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,6 +9,7 @@ coveralls==3.3.1 flake8==6.0.0 flake8-tidy-imports==4.8.0 +ruff==0.0.251 mypy==1.0.1 pre-commit==3.0.4 pytest==7.2.1 From b4ea37d59862ef5769468a0749c5f4e4ee4254da Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Feb 2023 21:08:17 +0100 Subject: [PATCH 18/27] Remove flake8 in favor of ruff --- .github/workflows/ci.yml | 12 ------------ CONTRIBUTING.md | 11 ++++++----- docs/developer.md | 2 +- environment.yml | 3 +-- requirements-dev.txt | 2 -- setup.py | 2 -- 6 files changed, 8 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cff9ef7a..e00b9040b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,10 +90,6 @@ jobs: freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 6 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - - name: Flake8 - run: | - flake8 - - name: Sort imports (isort) run: | isort --check . @@ -190,10 +186,6 @@ jobs: freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - - name: Flake8 - run: | - flake8 - - name: Sort imports (isort) run: | isort --check . @@ -256,10 +248,6 @@ jobs: freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - - name: Flake8 - run: | - flake8 - - name: Ruff - linting run: | ruff check . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b4e0bc024..040aae39c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,16 +45,17 @@ pytest tests/test_.py::test_ ### 2. Test if your code is PEP8 compliant -#### Run Flake8 +#### Run Ruff ```bash -flake8 freqtrade tests scripts +ruff . ``` -We receive a lot of code that fails the `flake8` checks. +We receive a lot of code that fails the `ruff` checks. To help with that, we encourage you to install the git pre-commit -hook that will warn you when you try to commit code that fails these checks. -Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using-hooks.html). +hook that will warn you when you try to commit code that fails these checks. + +you can manually run pre-commit with `pre-commit run -a`. ##### Additional styles applied diff --git a/docs/developer.md b/docs/developer.md index 0546c20e9..1bc75551f 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -24,7 +24,7 @@ This will spin up a local server (usually on port 8000) so you can see if everyt To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ". Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`. -This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`. +This will install all required tools for development, including `pytest`, `ruff`, `mypy`, and `coveralls`. Then install the git hook scripts by running `pre-commit install`, so your changes will be verified locally before committing. This avoids a lot of waiting for CI already, as some basic formatting checks are done locally on your machine. diff --git a/environment.yml b/environment.yml index 5b039e7f7..171e7c3da 100644 --- a/environment.yml +++ b/environment.yml @@ -41,7 +41,6 @@ dependencies: # 2/4 req dev - coveralls - - flake8 - mypy - pytest - pytest-asyncio @@ -70,6 +69,6 @@ dependencies: - tables - pytest-random-order - ccxt - - flake8-tidy-imports + - ruff - -e . # - python-rapidjso diff --git a/requirements-dev.txt b/requirements-dev.txt index 287cb8ae9..c23447694 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,8 +7,6 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -flake8==6.0.0 -flake8-tidy-imports==4.8.0 ruff==0.0.251 mypy==1.0.1 pre-commit==3.0.4 diff --git a/setup.py b/setup.py index 30aacc3f2..edd7b243b 100644 --- a/setup.py +++ b/setup.py @@ -32,8 +32,6 @@ hdf5 = [ develop = [ 'coveralls', - 'flake8', - 'flake8-tidy-imports', 'mypy', 'pytest', 'pytest-asyncio', From bf968a9fd878cfdb37fe7b0a5865f82767921cf6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Feb 2023 06:51:03 +0100 Subject: [PATCH 19/27] Use actions as documented --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e00b9040b..17c0efd6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,9 +94,9 @@ jobs: run: | isort --check . - - name: Ruff - linting + - name: Run Ruff run: | - ruff check . + ruff check --format=github . - name: Mypy run: | @@ -190,9 +190,9 @@ jobs: run: | isort --check . - - name: Ruff - linting + - name: Run Ruff run: | - ruff check . + ruff check --format=github . - name: Mypy run: | @@ -248,9 +248,9 @@ jobs: freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - - name: Ruff - linting + - name: Run Ruff run: | - ruff check . + ruff check --format=github . - name: Mypy run: | From 6b829d839b93d5a793a7ff70bfea632ce6a05655 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Feb 2023 07:12:54 +0100 Subject: [PATCH 20/27] Improve ruff config --- pyproject.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7da2a2200..8a7750731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,11 +62,11 @@ ignore = ["freqtrade/vendor/**"] line-length = 100 extend-exclude = [".env"] extend-select = [ - "TID", - "EXE", - "YTT", - # "DTZ", - # "RSE", - # "TCH", - # "PTH", + "TID", # flake8-tidy-imports + # "EXE", # flake8-executable + "YTT", # flake8-2020 + # "DTZ", # flake8-datetimez + # "RSE", # flake8-raise + # "TCH", # flake8-type-checking + # "PTH", # flake8-use-pathlib ] From c6455c41319736e95784832101a954a9e90d0fda Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 13:39:48 +0100 Subject: [PATCH 21/27] Pin scikit-learn to <1.2.0 for conda as well closes #8223 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 5b039e7f7..afe7a7ed7 100644 --- a/environment.yml +++ b/environment.yml @@ -54,7 +54,7 @@ dependencies: # 3/4 req hyperopt - scipy - - scikit-learn + - scikit-learn<1.2.0 - filelock - scikit-optimize - progressbar2 From dc2cfee056310c5a52bb00c2fdf280b6a70c026b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 13:49:16 +0100 Subject: [PATCH 22/27] Don't request sorted candles from HitBTC. Apparently hitBTC cannot properly handle this anymore. closes #8214 --- freqtrade/exchange/hitbtc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/exchange/hitbtc.py b/freqtrade/exchange/hitbtc.py index a48c9a198..bc4c7aa81 100644 --- a/freqtrade/exchange/hitbtc.py +++ b/freqtrade/exchange/hitbtc.py @@ -19,5 +19,4 @@ class Hitbtc(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, - "ohlcv_params": {"sort": "DESC"} } From 563742f13ce25e1399734d830d77ecba5d609772 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 13:34:19 +0100 Subject: [PATCH 23/27] Fix enum behavior for python 3.11 closes #8221 closes #8217 --- freqtrade/enums/candletype.py | 3 +++ freqtrade/enums/rpcmessagetype.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/freqtrade/enums/candletype.py b/freqtrade/enums/candletype.py index 9d05ff6d7..dcb9f1448 100644 --- a/freqtrade/enums/candletype.py +++ b/freqtrade/enums/candletype.py @@ -13,6 +13,9 @@ class CandleType(str, Enum): FUNDING_RATE = "funding_rate" # BORROW_RATE = "borrow_rate" # * unimplemented + def __str__(self): + return f"{self.name.lower()}" + @staticmethod def from_string(value: str) -> 'CandleType': if not value: diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py index 2453d16d9..404c75401 100644 --- a/freqtrade/enums/rpcmessagetype.py +++ b/freqtrade/enums/rpcmessagetype.py @@ -37,5 +37,8 @@ class RPCRequestType(str, Enum): WHITELIST = 'whitelist' ANALYZED_DF = 'analyzed_df' + def __str__(self): + return self.value + NO_ECHO_MESSAGES = (RPCMessageType.ANALYZED_DF, RPCMessageType.WHITELIST, RPCMessageType.NEW_CANDLE) From be352ae01499f794ced15af4cfec905f773c4407 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 13:46:14 +0100 Subject: [PATCH 24/27] Update more enums --- freqtrade/enums/signaltype.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/freqtrade/enums/signaltype.py b/freqtrade/enums/signaltype.py index f706fd4dc..b5af1f1b2 100644 --- a/freqtrade/enums/signaltype.py +++ b/freqtrade/enums/signaltype.py @@ -10,6 +10,9 @@ class SignalType(Enum): ENTER_SHORT = "enter_short" EXIT_SHORT = "exit_short" + def __str__(self): + return f"{self.name.lower()}" + class SignalTagType(Enum): """ @@ -18,7 +21,13 @@ class SignalTagType(Enum): ENTER_TAG = "enter_tag" EXIT_TAG = "exit_tag" + def __str__(self): + return f"{self.name.lower()}" + class SignalDirection(str, Enum): LONG = 'long' SHORT = 'short' + + def __str__(self): + return f"{self.name.lower()}" From ff3aa7c1a996e99191b2e315e93bf3300ff91e6b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 16:10:24 +0100 Subject: [PATCH 25/27] Bump Version to 2023.3.dev --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index db339bea3..6ba045adf 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.2.dev' +__version__ = '2023.3.dev' if 'dev' in __version__: from pathlib import Path From c8a4a773ee192a7eb495da728181dc738f29ef69 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 Feb 2023 18:15:27 +0100 Subject: [PATCH 26/27] Fix _pairs_last_refresh_time storing the wrong date Depending on the drop_incomplete settings, this can lead to implicit bugs --- freqtrade/exchange/exchange.py | 7 +++++-- tests/exchange/test_exchange.py | 10 +++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 0cac411c7..cdbda1506 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1961,7 +1961,8 @@ class Exchange: cache: bool, drop_incomplete: bool) -> DataFrame: # keeping last candle time as last refreshed time of the pair if ticks and cache: - self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000 + idx = -2 if drop_incomplete and len(ticks) > 1 else -1 + self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[idx][0] // 1000 # keeping parsed dataframe in cache ohlcv_df = ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=drop_incomplete) @@ -2034,7 +2035,9 @@ class Exchange: # Timeframe in seconds interval_in_sec = timeframe_to_seconds(timeframe) plr = self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0) + interval_in_sec - return plr < arrow.utcnow().int_timestamp + # current,active candle open date + now = int(timeframe_to_prev_date(timeframe).timestamp()) + return plr < now @retrier_async async def _async_get_candle_history( diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 13613df37..df878dbd3 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2215,7 +2215,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach assert len(res[pair1]) == 99 assert len(res[pair2]) == 99 assert exchange._klines - assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000 + assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000 exchange._api_async.fetch_ohlcv.reset_mock() # Returned from cache @@ -2224,7 +2224,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach assert len(res) == 2 assert len(res[pair1]) == 99 assert len(res[pair2]) == 99 - assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000 + assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000 # Move time 1 candle further but result didn't change yet time_machine.move_to(start + timedelta(hours=101)) @@ -2234,7 +2234,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach assert len(res[pair1]) == 99 assert len(res[pair2]) == 99 assert res[pair2].at[0, 'open'] - assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000 + assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000 refresh_pior = exchange._pairs_last_refresh_time[pair1] # New candle on exchange - return 100 candles - but skip one candle so we actually get 2 candles @@ -2252,8 +2252,8 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach assert res[pair2].at[0, 'open'] assert refresh_pior != exchange._pairs_last_refresh_time[pair1] - assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000 - assert exchange._pairs_last_refresh_time[pair2] == ohlcv[-1][0] // 1000 + assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000 + assert exchange._pairs_last_refresh_time[pair2] == ohlcv[-2][0] // 1000 exchange._api_async.fetch_ohlcv.reset_mock() # Retry same call - from cache From 305eda74e2e8ccff87b727cc6c034b4b5ded19b0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 20:50:26 +0100 Subject: [PATCH 27/27] Enable Complexity for ruff --- freqtrade/commands/data_commands.py | 22 +++++++++++++--------- pyproject.toml | 7 +++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 2cd736b3e..1e74e1036 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta from typing import Any, Dict, List from freqtrade.configuration import TimeRange, setup_utils_configuration -from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.constants import DATETIME_PRINT_FORMAT, Config from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) @@ -20,15 +20,24 @@ from freqtrade.util.binance_mig import migrate_binance_futures_data logger = logging.getLogger(__name__) +def _data_download_sanity(config: Config) -> None: + if 'days' in config and 'timerange' in config: + raise OperationalException("--days and --timerange are mutually exclusive. " + "You can only specify one or the other.") + + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + + def start_download_data(args: Dict[str, Any]) -> None: """ Download data (former download_backtest_data.py script) """ config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - if 'days' in config and 'timerange' in config: - raise OperationalException("--days and --timerange are mutually exclusive. " - "You can only specify one or the other.") + _data_download_sanity(config) timerange = TimeRange() if 'days' in config: time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d") @@ -40,11 +49,6 @@ def start_download_data(args: Dict[str, Any]) -> None: # Remove stake-currency to skip checks which are not relevant for datadownload config['stake_currency'] = '' - if 'pairs' not in config: - raise OperationalException( - "Downloading data requires a list of pairs. " - "Please check the documentation on how to configure this.") - pairs_not_available: List[str] = [] # Init exchange diff --git a/pyproject.toml b/pyproject.toml index 8a7750731..698a621b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,11 @@ ignore = ["freqtrade/vendor/**"] [tool.ruff] line-length = 100 extend-exclude = [".env"] +target-version = "py38" extend-select = [ + "C90", # mccabe + # "N", # pep8-naming + # "UP", # pyupgrade "TID", # flake8-tidy-imports # "EXE", # flake8-executable "YTT", # flake8-2020 @@ -70,3 +74,6 @@ extend-select = [ # "TCH", # flake8-type-checking # "PTH", # flake8-use-pathlib ] + +[tool.ruff.mccabe] +max-complexity = 12