From 1c6f96657974f69fbf97ea135740f180393d0239 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 19 Aug 2022 15:03:03 +0200 Subject: [PATCH 1/9] Hyperopt: simplify parameter "can_optimize" handling --- freqtrade/strategy/parameters.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/parameters.py b/freqtrade/strategy/parameters.py index 83dd41de9..e64a1e4c0 100644 --- a/freqtrade/strategy/parameters.py +++ b/freqtrade/strategy/parameters.py @@ -57,6 +57,12 @@ class BaseParameter(ABC): Get-space - will be used by Hyperopt to get the hyperopt Space """ + def can_optimize(self): + return ( + self.in_space + and self.optimize + ) + class NumericParameter(BaseParameter): """ Internal parameter used for Numeric purposes """ @@ -133,7 +139,7 @@ class IntParameter(NumericParameter): Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid calculating 100ds of indicators. """ - if self.in_space and self.optimize: + if self.can_optimize(): # Scikit-optimize ranges are "inclusive", while python's "range" is exclusive return range(self.low, self.high + 1) else: @@ -212,7 +218,7 @@ class DecimalParameter(NumericParameter): Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid calculating 100ds of indicators. """ - if self.in_space and self.optimize: + if self.can_optimize(): low = int(self.low * pow(10, self._decimals)) high = int(self.high * pow(10, self._decimals)) + 1 return [round(n * pow(0.1, self._decimals), self._decimals) for n in range(low, high)] @@ -261,7 +267,7 @@ class CategoricalParameter(BaseParameter): Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid calculating 100ds of indicators. """ - if self.in_space and self.optimize: + if self.can_optimize(): return self.opt_range else: return [self.value] From 08ef5ad2d8f12100e17beec5c4c23d26d5e51434 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 19 Aug 2022 15:11:43 +0200 Subject: [PATCH 2/9] Add HyperoptState enum and container class --- freqtrade/enums/__init__.py | 1 + freqtrade/enums/hyperoptstate.py | 12 ++++++++++++ freqtrade/optimize/hyperopt_tools.py | 15 +++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 freqtrade/enums/hyperoptstate.py diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py index e50ebc4a4..d2f5474fc 100644 --- a/freqtrade/enums/__init__.py +++ b/freqtrade/enums/__init__.py @@ -3,6 +3,7 @@ from freqtrade.enums.backteststate import BacktestState from freqtrade.enums.candletype import CandleType from freqtrade.enums.exitchecktuple import ExitCheckTuple from freqtrade.enums.exittype import ExitType +from freqtrade.enums.hyperoptstate import HyperoptState from freqtrade.enums.marginmode import MarginMode from freqtrade.enums.ordertypevalue import OrderTypeValues from freqtrade.enums.rpcmessagetype import RPCMessageType diff --git a/freqtrade/enums/hyperoptstate.py b/freqtrade/enums/hyperoptstate.py new file mode 100644 index 000000000..332b3354d --- /dev/null +++ b/freqtrade/enums/hyperoptstate.py @@ -0,0 +1,12 @@ +from enum import Enum + + +class HyperoptState(Enum): + """ Hyperopt states """ + STARTUP = 1 + DATALOAD = 2 + INDICATORS = 3 + OPTIMIZE = 0 + + def __str__(self): + return f"{self.name.lower()}" diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index ab6ef013b..7f339eec1 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -13,6 +13,7 @@ from colorama import Fore, Style from pandas import isna, json_normalize from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES +from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2 from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs @@ -32,6 +33,20 @@ def hyperopt_serializer(x): return str(x) +class HyperoptStateContainer(): + """ Singleton class to track state of hyperopt""" + __state = HyperoptState.OPTIMIZE + + @classmethod + def set_state(cls, value: HyperoptState): + cls.__state = value + + @classmethod + @property + def state(cls) -> HyperoptState: + return cls.__state + + class HyperoptTools(): @staticmethod From 09f8904545b35454534cf2866216ec4e656c4381 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 19 Aug 2022 15:12:55 +0200 Subject: [PATCH 3/9] Extract analysis to separate method --- freqtrade/optimize/hyperopt.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index cbcf39131..ebeb7eb25 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -406,6 +406,14 @@ class Hyperopt: def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) + def advise_and_trim(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]: + preprocessed = self.backtesting.strategy.advise_all_indicators(data) + + # Trim startup period from analyzed dataframe to get correct dates for output. + processed = trim_dataframes(preprocessed, self.timerange, self.backtesting.required_startup) + self.min_date, self.max_date = get_timerange(processed) + return processed + def prepare_hyperopt_data(self) -> None: data, timerange = self.backtesting.load_bt_data() self.backtesting.load_bt_data_detail() @@ -413,9 +421,7 @@ class Hyperopt: preprocessed = self.backtesting.strategy.advise_all_indicators(data) - # Trim startup period from analyzed dataframe to get correct dates for output. - processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup) - self.min_date, self.max_date = get_timerange(processed) + preprocessed = self.advise_and_trim(data) logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} ' From bc359675a2bebf55e79e72d3e6c5c54bf519e159 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 19 Aug 2022 15:19:43 +0200 Subject: [PATCH 4/9] Add --analyze-per-epoch - moving populate_analysis to the epoch process --- docs/hyperopt.md | 10 +++++-- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 7 +++++ freqtrade/configuration/configuration.py | 3 +++ freqtrade/optimize/hyperopt.py | 34 +++++++++++++++++------- freqtrade/strategy/parameters.py | 4 +++ 6 files changed, 48 insertions(+), 12 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index c9ec30056..a07bab9de 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -40,7 +40,8 @@ pip install -r requirements-hyperopt.txt ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] - [--recursive-strategy-search] [-i TIMEFRAME] + [--recursive-strategy-search] [--freqaimodel NAME] + [--freqaimodel-path PATH] [-i TIMEFRAME] [--timerange TIMERANGE] [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] @@ -53,7 +54,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] [--hyperopt-loss NAME] [--disable-param-export] - [--ignore-missing-spaces] + [--ignore-missing-spaces] [--analyze-per-epoch] optional arguments: -h, --help show this help message and exit @@ -129,6 +130,7 @@ optional arguments: --ignore-missing-spaces, --ignore-unparameterized-spaces Suppress errors for any requested Hyperopt spaces that do not contain any parameters. + --analyze-per-epoch Run populate_indicators once per epoch. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -154,6 +156,10 @@ Strategy arguments: --recursive-strategy-search Recursively search for a strategy in the strategies folder. + --freqaimodel NAME Specify a custom freqaimodels. + --freqaimodel-path PATH + Specify additional lookup path for freqaimodels. + ``` ### Hyperopt checklist diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 48a423be4..05a6a2ca3 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -34,7 +34,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", "hyperopt_loss", "disableparamexport", - "hyperopt_ignore_missing_space"] + "hyperopt_ignore_missing_space", "analyze_per_epoch"] ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index f85b75af1..51a501d7c 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -255,6 +255,13 @@ AVAILABLE_CLI_OPTIONS = { nargs='+', default='default', ), + "analyze_per_epoch": Arg( + '--analyze-per-epoch', + help='Run populate_indicators once per epoch.', + action='store_true', + default=False, + ), + "print_all": Arg( '--print-all', help='Print all results, not only the best ones.', diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index b4f36aa3c..41b31b022 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -302,6 +302,9 @@ class Configuration: self._args_to_config(config, argname='spaces', logstring='Parameter -s/--spaces detected: {}') + self._args_to_config(config, argname='analyze_per_epoch', + logstring='Parameter --analyze-per-epoch detected.') + self._args_to_config(config, argname='print_all', logstring='Parameter --print-all detected ...') diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ebeb7eb25..fea2a672f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -24,13 +24,15 @@ from pandas import DataFrame from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange +from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, file_dump_json, plural from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss -from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer +from freqtrade.optimize.hyperopt_tools import (HyperoptStateContainer, HyperoptTools, + hyperopt_serializer) from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver @@ -74,10 +76,14 @@ class Hyperopt: self.dimensions: List[Dimension] = [] self.config = config + self.min_date: datetime + self.max_date: datetime self.backtesting = Backtesting(self.config) self.pairlist = self.backtesting.pairlists.whitelist self.custom_hyperopt: HyperOptAuto + self.analyze_per_epoch = self.config.get('analyze_per_epoch', False) + HyperoptStateContainer.set_state(HyperoptState.STARTUP) if not self.config.get('hyperopt'): self.custom_hyperopt = HyperOptAuto(self.config) @@ -290,6 +296,7 @@ class Hyperopt: Called once per epoch to optimize whatever is configured. Keep this function as optimized as possible! """ + HyperoptStateContainer.set_state(HyperoptState.OPTIMIZE) backtest_start_time = datetime.now(timezone.utc) params_dict = self._get_params_dict(self.dimensions, raw_params) @@ -321,6 +328,10 @@ class Hyperopt: with self.data_pickle_file.open('rb') as f: processed = load(f, mmap_mode='r') + if self.analyze_per_epoch: + # Data is not yet analyzed, rerun populate_indicators. + processed = self.advise_and_trim(processed) + bt_results = self.backtesting.backtest( processed=processed, start_date=self.min_date, @@ -415,19 +426,24 @@ class Hyperopt: return processed def prepare_hyperopt_data(self) -> None: - data, timerange = self.backtesting.load_bt_data() + HyperoptStateContainer.set_state(HyperoptState.DATALOAD) + data, self.timerange = self.backtesting.load_bt_data() self.backtesting.load_bt_data_detail() logger.info("Dataload complete. Calculating indicators") - preprocessed = self.backtesting.strategy.advise_all_indicators(data) + if not self.analyze_per_epoch: + HyperoptStateContainer.set_state(HyperoptState.INDICATORS) - preprocessed = self.advise_and_trim(data) + preprocessed = self.advise_and_trim(data) - logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'({(self.max_date - self.min_date).days} days)..') - # Store non-trimmed data - will be trimmed after signal generation. - dump(preprocessed, self.data_pickle_file) + logger.info(f'Hyperopting with data from ' + f'{self.min_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'({(self.max_date - self.min_date).days} days)..') + # Store non-trimmed data - will be trimmed after signal generation. + dump(preprocessed, self.data_pickle_file) + else: + dump(data, self.data_pickle_file) def get_asked_points(self, n_points: int) -> Tuple[List[List[Any]], List[bool]]: """ diff --git a/freqtrade/strategy/parameters.py b/freqtrade/strategy/parameters.py index e64a1e4c0..c6037ae0b 100644 --- a/freqtrade/strategy/parameters.py +++ b/freqtrade/strategy/parameters.py @@ -7,6 +7,9 @@ from abc import ABC, abstractmethod from contextlib import suppress from typing import Any, Optional, Sequence, Union +from freqtrade.enums.hyperoptstate import HyperoptState +from freqtrade.optimize.hyperopt_tools import HyperoptStateContainer + with suppress(ImportError): from skopt.space import Integer, Real, Categorical @@ -61,6 +64,7 @@ class BaseParameter(ABC): return ( self.in_space and self.optimize + and HyperoptStateContainer.state != HyperoptState.OPTIMIZE ) From 733f716819b2c9ecaf303a357744005eba38ff5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 19 Aug 2022 15:22:43 +0200 Subject: [PATCH 5/9] Update documentation --- docs/hyperopt.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index a07bab9de..d230c091e 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -191,7 +191,7 @@ Rarely you may also need to create a [nested class](advanced-hyperopt.md#overrid ### Hyperopt execution logic -Hyperopt will first load your data into memory and will then run `populate_indicators()` once per Pair to generate all indicators. +Hyperopt will first load your data into memory and will then run `populate_indicators()` once per Pair to generate all indicators, unless `--analyze-per-epoch` is specified. Hyperopt will then spawn into different processes (number of processors, or `-j `), and run backtesting over and over again, changing the parameters that are part of the `--spaces` defined. @@ -434,7 +434,8 @@ While this strategy is most likely too simple to provide consistent profit, it s ??? Hint "Performance tip" By doing the calculation of all possible indicators in `populate_indicators()`, the calculation of the indicator happens only once for every parameter. While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values). - You should however try to use space ranges as small as possible. Every new column will require more memory, and every possibility hyperopt can try will increase the search space. + As this also has Performance implications, hyperopt provides `--analyze-per-epoch` - which will move the execution of `populate_indicators()` to the epoch process. This will implicitly also change the `.range` functionality to only return the actually used value. + You should however try to use space ranges as small as possible. ## Optimizing protections @@ -885,6 +886,7 @@ To combat these, you have multiple options: * Avoid using `--timeframe-detail` (this loads a lot of additional data into memory). * Reduce the number of parallel processes (`-j `). * Increase the memory of your machine. +* Use `--analyze-per-epoch` if you're using a lot of parameters with `.range` functionality. ## The objective has been evaluated at this point before. From b9d48c32782040ebe0754f4b11cf1dd481264235 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 19 Aug 2022 15:40:06 +0200 Subject: [PATCH 6/9] use numbers in HyperoptState properly ... --- freqtrade/enums/hyperoptstate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/enums/hyperoptstate.py b/freqtrade/enums/hyperoptstate.py index 332b3354d..6716e123a 100644 --- a/freqtrade/enums/hyperoptstate.py +++ b/freqtrade/enums/hyperoptstate.py @@ -6,7 +6,7 @@ class HyperoptState(Enum): STARTUP = 1 DATALOAD = 2 INDICATORS = 3 - OPTIMIZE = 0 + OPTIMIZE = 4 def __str__(self): return f"{self.name.lower()}" From 665cf4431de06e86d3edf96114c8e3b33341cc07 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 19 Aug 2022 15:49:31 +0200 Subject: [PATCH 7/9] Add explicit test cov. for .range behavior --- tests/strategy/test_interface.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 83f7d19b7..65ee05d71 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -12,7 +12,9 @@ from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import load_data from freqtrade.enums import ExitCheckTuple, ExitType, SignalDirection +from freqtrade.enums.hyperoptstate import HyperoptState from freqtrade.exceptions import OperationalException, StrategyError +from freqtrade.optimize.hyperopt_tools import HyperoptStateContainer from freqtrade.optimize.space import SKDecimal from freqtrade.persistence import PairLocks, Trade from freqtrade.resolvers import StrategyResolver @@ -859,7 +861,9 @@ def test_strategy_safe_wrapper_trade_copy(fee): def test_hyperopt_parameters(): + HyperoptStateContainer.set_state(HyperoptState.INDICATORS) from skopt.space import Categorical, Integer, Real + with pytest.raises(OperationalException, match=r"Name is determined.*"): IntParameter(low=0, high=5, default=1, name='hello') @@ -937,6 +941,12 @@ def test_hyperopt_parameters(): assert list(boolpar.range) == [True, False] + HyperoptStateContainer.set_state(HyperoptState.OPTIMIZE) + assert len(list(intpar.range)) == 1 + assert len(list(fltpar.range)) == 1 + assert len(list(catpar.range)) == 1 + assert len(list(boolpar.range)) == 1 + def test_auto_hyperopt_interface(default_conf): default_conf.update({'strategy': 'HyperoptableStrategyV2'}) From aa3da092a0262a1efbf6387fbcadc928b5b04306 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 Aug 2022 10:55:52 +0200 Subject: [PATCH 8/9] Dont' use classProperty - that's not supported on 3.8 --- freqtrade/optimize/hyperopt_tools.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 7f339eec1..9b022d519 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -35,16 +35,11 @@ def hyperopt_serializer(x): class HyperoptStateContainer(): """ Singleton class to track state of hyperopt""" - __state = HyperoptState.OPTIMIZE + state: HyperoptState = HyperoptState.OPTIMIZE @classmethod def set_state(cls, value: HyperoptState): - cls.__state = value - - @classmethod - @property - def state(cls) -> HyperoptState: - return cls.__state + cls.state = value class HyperoptTools(): From c3e74e6e8d59f002185112547f88ddb187484bee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Aug 2022 08:55:29 +0200 Subject: [PATCH 9/9] Improve doc wording --- docs/hyperopt.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index d230c091e..6b6c2a772 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -432,10 +432,10 @@ While this strategy is most likely too simple to provide consistent profit, it s `range` property may also be used with `DecimalParameter` and `CategoricalParameter`. `RealParameter` does not provide this property due to infinite search space. ??? Hint "Performance tip" - By doing the calculation of all possible indicators in `populate_indicators()`, the calculation of the indicator happens only once for every parameter. - While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values). - As this also has Performance implications, hyperopt provides `--analyze-per-epoch` - which will move the execution of `populate_indicators()` to the epoch process. This will implicitly also change the `.range` functionality to only return the actually used value. - You should however try to use space ranges as small as possible. + During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value. This will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues. + + In either case, you should try to use space ranges as small as possible this will improve CPU/RAM usage in both scenarios. + ## Optimizing protections