From 7c8e26c717fd61550d8ffe3077b2c4d3d0a7dcb4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 23 Apr 2019 00:30:09 +0300 Subject: [PATCH] -j/--job-workers option added for controlling the number of joblib parallel worker processes used in hyperopt docs refreshed --- docs/bot-usage.md | 19 ++++++++++++++++--- freqtrade/arguments.py | 11 +++++++++++ freqtrade/configuration.py | 18 ++++-------------- freqtrade/optimize/hyperopt.py | 16 ++++++++++------ 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 55988985a..9261294e1 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -206,8 +206,11 @@ to find optimal parameter values for your stategy. ``` usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE] - [--customhyperopt NAME] [--eps] [--dmmp] [-e INT] - [-s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]] + [--max_open_trades MAX_OPEN_TRADES] + [--stake_amount STAKE_AMOUNT] [--customhyperopt NAME] + [--eps] [--dmmp] [-e INT] + [-s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]] + [--print-all] [-j JOBS] optional arguments: -h, --help show this help message and exit @@ -215,6 +218,10 @@ optional arguments: Specify ticker interval (1m, 5m, 30m, 1h, 1d). --timerange TIMERANGE Specify what timerange of data to use. + --max_open_trades MAX_OPEN_TRADES + Specify max_open_trades to use. + --stake_amount STAKE_AMOUNT + Specify stake_amount. --customhyperopt NAME Specify hyperopt class name (default: DefaultHyperOpts). @@ -229,7 +236,13 @@ optional arguments: -s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...], --spaces {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate list. Default: all. - + --print-all Print all results, not only the best ones. + -j JOBS, --job-workers JOBS + The number of concurrently running jobs for + hyperoptimization (hyperopt worker processes). If -1 + (default), all CPUs are used, for -2, all CPUs but one + are used, etc. If 1 is given, no parallel computing + code is used at all. ``` ## Edge commands diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 96f080bd2..3631e6615 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -315,6 +315,17 @@ class Arguments(object): dest='print_all', default=False ) + parser.add_argument( + '-j', '--job-workers', + help='The number of concurrently running jobs for hyperoptimization ' + '(hyperopt worker processes). ' + 'If -1 (default), all CPUs are used, for -2, all CPUs but one are used, etc. ' + 'If 1 is given, no parallel computing code is used at all.', + dest='hyperopt_jobs', + default=-1, + type=int, + metavar='JOBS', + ) def _build_subcommands(self) -> None: """ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 65a8d644e..a13c24f6a 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -208,17 +208,14 @@ class Configuration(object): logger.info('Parameter -i/--ticker-interval detected ...') logger.info('Using ticker_interval: %s ...', config.get('ticker_interval')) - # If -l/--live is used we add it to the configuration if 'live' in self.args and self.args.live: config.update({'live': True}) logger.info('Parameter -l/--live detected ...') - # If --enable-position-stacking is used we add it to the configuration if 'position_stacking' in self.args and self.args.position_stacking: config.update({'position_stacking': True}) logger.info('Parameter --enable-position-stacking detected ...') - # If --disable-max-market-positions or --max_open_trades is used we update configuration if 'use_max_market_positions' in self.args and not self.args.use_max_market_positions: config.update({'use_max_market_positions': False}) logger.info('Parameter --disable-max-market-positions detected ...') @@ -230,25 +227,21 @@ class Configuration(object): else: logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) - # If --stake_amount is used we update configuration if 'stake_amount' in self.args and self.args.stake_amount: config.update({'stake_amount': self.args.stake_amount}) logger.info('Parameter --stake_amount detected, overriding stake_amount to: %s ...', config.get('stake_amount')) - # If --timerange is used we add it to the configuration if 'timerange' in self.args and self.args.timerange: config.update({'timerange': self.args.timerange}) logger.info('Parameter --timerange detected: %s ...', self.args.timerange) - # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self._create_datadir(config, self.args.datadir)}) else: config.update({'datadir': self._create_datadir(config, None)}) logger.info('Using data folder: %s ...', config.get('datadir')) - # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: config.update({'refresh_pairs': True}) logger.info('Parameter -r/--refresh-pairs-cached detected ...') @@ -261,12 +254,10 @@ class Configuration(object): config.update({'ticker_interval': self.args.ticker_interval}) logger.info('Overriding ticker interval with Command line argument') - # If --export is used we add it to the configuration if 'export' in self.args and self.args.export: config.update({'export': self.args.export}) logger.info('Parameter --export detected: %s ...', self.args.export) - # If --export-filename is used we add it to the configuration if 'export' in config and 'exportfilename' in self.args and self.args.exportfilename: config.update({'exportfilename': self.args.exportfilename}) logger.info('Storing backtest results to %s ...', self.args.exportfilename) @@ -279,12 +270,10 @@ class Configuration(object): :return: configuration as dictionary """ - # If --timerange is used we add it to the configuration if 'timerange' in self.args and self.args.timerange: config.update({'timerange': self.args.timerange}) logger.info('Parameter --timerange detected: %s ...', self.args.timerange) - # If --timerange is used we add it to the configuration if 'stoploss_range' in self.args and self.args.stoploss_range: txt_range = eval(self.args.stoploss_range) config['edge'].update({'stoploss_range_min': txt_range[0]}) @@ -292,7 +281,6 @@ class Configuration(object): config['edge'].update({'stoploss_range_step': txt_range[2]}) logger.info('Parameter --stoplosses detected: %s ...', self.args.stoploss_range) - # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: config.update({'refresh_pairs': True}) logger.info('Parameter -r/--refresh-pairs-cached detected ...') @@ -309,13 +297,11 @@ class Configuration(object): # Add the hyperopt file to use config.update({'hyperopt': self.args.hyperopt}) - # If --epochs is used we add it to the configuration if 'epochs' in self.args and self.args.epochs: config.update({'epochs': self.args.epochs}) logger.info('Parameter --epochs detected ...') logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs')) - # If --spaces is used we add it to the configuration if 'spaces' in self.args and self.args.spaces: config.update({'spaces': self.args.spaces}) logger.info('Parameter -s/--spaces detected: %s', config.get('spaces')) @@ -324,6 +310,10 @@ class Configuration(object): config.update({'print_all': self.args.print_all}) logger.info('Parameter --print-all detected: %s', config.get('print_all')) + if 'hyperopt_jobs' in self.args and self.args.hyperopt_jobs: + config.update({'hyperopt_jobs': self.args.hyperopt_jobs}) + logger.info('Parameter -j/--job-workers detected: %s', config.get('hyperopt_jobs')) + return config def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]: diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b37027244..2bcfdd499 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -270,21 +270,25 @@ class Hyperopt(Backtesting): cpus = multiprocessing.cpu_count() logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') + config_jobs = self.config.get('hyperopt_jobs', -1) + logger.info(f'Number of parallel jobs set as: {config_jobs}') - opt = self.get_optimizer(cpus) - EVALS = max(self.total_tries // cpus, 1) + opt = self.get_optimizer(config_jobs) try: - with Parallel(n_jobs=cpus) as parallel: + with Parallel(n_jobs=config_jobs) as parallel: + jobs = parallel._effective_n_jobs() + logger.info(f'Effective number of parallel workers used: {jobs}') + EVALS = max(self.total_tries // jobs, 1) for i in range(EVALS): - asked = opt.ask(n_points=cpus) + asked = opt.ask(n_points=jobs) f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) self.trials += f_val - for j in range(cpus): + for j in range(jobs): self.log_results({ 'loss': f_val[j]['loss'], - 'current_tries': i * cpus + j, + 'current_tries': i * jobs + j, 'total_tries': self.total_tries, 'result': f_val[j]['result'], })