Merge commit 'e3d222912dfd775b7456a44d6d6055430711f251' into feature/objectify

docs/hyperopt.md

@@ -51,12 +51,12 @@ def populate_buy_trend(dataframe: DataFrame) -> DataFrame:
     return dataframe
 ```
 
-Your hyperopt file must contains `guards` to find the right value for 
+Your hyperopt file must contain `guards` to find the right value for 
 `(dataframe['adx'] > 65)` & and `(dataframe['plus_di'] > 0.5)`. That 
 means you will need to enable/disable triggers.
  
 In our case the `SPACE` and `populate_buy_trend` in your strategy file 
-will be look like:
+will look like:
 ```python
 space = {
     'rsi': hp.choice('rsi', [
@@ -105,7 +105,7 @@ def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
 
 
 ### 2. Update the hyperopt config file
-Hyperopt is using a dedicated config file. At this moment hyperopt 
+Hyperopt is using a dedicated config file. Currently hyperopt 
 cannot use your config file. It is also made on purpose to allow you
 testing your strategy with different configurations.
 
@@ -127,19 +127,21 @@ If it's a guard, you will add a line like this:
     {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)}
 ]),
 ```
-This says, "*one of guards is RSI, it can have two values, enabled or 
+This says, "*one of the guards is RSI, it can have two values, enabled or 
 disabled. If it is enabled, try different values for it between 20 and 40*".
 
 So, the part of the strategy builder using the above setting looks like 
 this:
+
 ```
 if params['rsi']['enabled']:
     conditions.append(dataframe['rsi'] < params['rsi']['value'])
 ```
+
 It checks if Hyperopt wants the RSI guard to be enabled for this 
 round `params['rsi']['enabled']` and if it is, then it will add a 
-condition that says RSI must be < than the value hyperopt picked 
-for this evaluation, that is given in the `params['rsi']['value']`.
+condition that says RSI must be smaller than the value hyperopt picked 
+for this evaluation, which is given in the `params['rsi']['value']`.
 
 That's it. Now you can add new parts of strategies to Hyperopt and it 
 will try all the combinations with all different values in the search 
@@ -148,8 +150,7 @@ for best working algo.
 
 ### Add a new Indicators
 If you want to test an indicator that isn't used by the bot currently, 
-you need to add it to your strategy file (example: [user_data/strategies/test_strategy.py](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py))
-inside the `populate_indicators()` method.
+you need to add it to the `populate_indicators()` method in `hyperopt.py`.
 
 ## Execute Hyperopt
 Once you have updated your hyperopt configuration you can run it. 
@@ -158,17 +159,19 @@ it will take time you will have the result (more than 30 mins).
 
 We strongly recommend to use `screen` to prevent any connection loss.
 ```bash
-python3 ./freqtrade/main.py -c config.json hyperopt
+python3 ./freqtrade/main.py -c config.json hyperopt -e 5000
 ```
 
+The `-e` flag will set how many evaluations hyperopt will do. We recommend
+running at least several thousand evaluations.
+
 ### Execute hyperopt with different ticker-data source
-If you would like to learn parameters using an alternate ticke-data that
+If you would like to hyperopt parameters using an alternate ticker data that
 you have on-disk, use the `--datadir PATH` option. Default hyperopt will
 use data from directory `user_data/data`.
 
 ### Running hyperopt with smaller testset
-
-Use the --timeperiod argument to change how much of the testset
+Use the `--timeperiod` argument to change how much of the testset
 you want to use. The last N ticks/timeframes will be used.
 Example:
 
@@ -176,6 +179,21 @@ Example:
 python3 ./freqtrade/main.py hyperopt --timeperiod -200
 ```
 
+### Running hyperopt with smaller search space
+Use the `--spaces` argument to limit the search space used by hyperopt.
+Letting Hyperopt optimize everything is a huuuuge search space. Often it 
+might make more sense to start by just searching for initial buy algorithm. 
+Or maybe you just want to optimize your stoploss or roi table for that awesome 
+new buy strategy you have.
+
+Legal values are:
+
+- `all`: optimize everything
+- `buy`: just search for a new buy strategy
+- `roi`: just optimize the minimal profit table for your strategy
+- `stoploss`: search for the best stoploss value
+- space-separated list of any of the above values for example `--spaces roi stoploss`
+
 ### Hyperopt with MongoDB
 Hyperopt with MongoDB, is like Hyperopt under steroids. As you saw by
 executing the previous command is the execution takes a long time. 
@@ -267,7 +285,6 @@ customizable value.
 - You should **ignore** the guard "mfi" (`"mfi"` is `"enabled": false`)
 - and so on...
 
-
 You have to look inside your strategy file into `buy_strategy_generator()` 
 method, what those values match to.   
   
@@ -277,7 +294,7 @@ at `adx`-block, that translates to the following code block:
 (dataframe['adx'] > 15.0)
 ```
   
-So translating your whole hyperopt result to as the new buy-signal 
+Translating your whole hyperopt result to as the new buy-signal 
 would be the following:
 ```
 def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:

freqtrade/arguments.py

@@ -180,6 +180,15 @@ class Arguments(object):
             type=str,
             dest='timerange',
         )
+        parser.add_argument(
+            '-s', '--spaces',
+            help='Specify which parameters to hyperopt. Space separate list. \
+                  Default: %(default)s',
+            choices=['all', 'buy', 'roi', 'stoploss'],
+            default='all',
+            nargs='+',
+            dest='spaces',
+        )
 
     def _build_subcommands(self) -> None:
         """

freqtrade/configuration.py

@@ -156,6 +156,11 @@ class Configuration(object):
             config.update({'mongodb': self.args.mongodb})
             self.logger.info('Parameter --use-mongodb detected ...')
 
+        # If --spaces is used we add it to the configuration
+        if 'spaces' in self.args and self.args.spaces:
+            config.update({'spaces': self.args.spaces})
+            self.logger.info('Parameter -s/--spaces detected: %s', config.get('spaces'))
+
         return config
 
     def _validate_config(self, conf: Dict[str, Any]) -> Dict[str, Any]:

freqtrade/optimize/backtesting.py

@@ -33,7 +33,6 @@ class Backtesting(object):
         # Init the logger
         self.logging = Logger(name=__name__, level=config['loglevel'])
         self.logger = self.logging.get_logger()
-
         self.config = config
         self.analyze = None
         self.ticker_interval = None

freqtrade/optimize/hyperopt.py

@@ -335,16 +335,26 @@ class Hyperopt(Backtesting):
             ]),
         }
 
-    @staticmethod
-    def hyperopt_space() -> Dict[str, Any]:
+    def has_space(self, space) -> bool:
+        """
+        Tell if a space value is contained in the configuration
+        """
+        if space in self.config['spaces'] or 'all' in self.config['spaces']:
+            return True
+        return False
+
+    def hyperopt_space(self) -> Dict[str, Any]:
         """
         Return the space to use during Hyperopt
         """
-        return {
-            **Hyperopt.indicator_space(),
-            **Hyperopt.roi_space(),
-            **Hyperopt.stoploss_space()
-        }
+        spaces = {}
+        if self.has_space('buy'):
+            spaces = {**spaces, **Hyperopt.indicator_space()}
+        if self.has_space('roi'):
+            spaces = {**spaces, **Hyperopt.roi_space()}
+        if self.has_space('stoploss'):
+            spaces = {**spaces, **Hyperopt.stoploss_space()}
+        return spaces
 
     @staticmethod
     def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
@@ -427,13 +437,19 @@ class Hyperopt(Backtesting):
         if 'roi_t1' in params:
             self.analyze.strategy.minimal_roi = self.generate_roi_table(params)
 
-        self.populate_buy_trend = self.buy_strategy_generator(params)
+        if 'trigger' in params:
+            self.populate_buy_trend = self.buy_strategy_generator(params)
+
+        if 'stoploss' in params:
+            stoploss = params['stoploss']
+        else:
+            stoploss = self.analyze.strategy.stoploss
 
         results = self.backtest(
             {
                 'stake_amount': self.config['stake_amount'],
                 'processed': self.processed,
-                'stoploss': params['stoploss']
+                'stoploss': stoploss
             }
         )
         result_explanation = self.format_results(results)
@@ -491,7 +507,8 @@ class Hyperopt(Backtesting):
             timerange=timerange
         )
 
-        self.analyze.populate_indicators = Hyperopt.populate_indicators
+        if self.has_space('buy'):
+            self.analyze.populate_indicators = Hyperopt.populate_indicators
         self.processed = self.tickerdata_to_dataframe(data)
 
         if self.config.get('mongodb'):

freqtrade/tests/optimize/test_hyperopt.py

@@ -126,6 +126,7 @@ def test_fmin_best_results(mocker, default_conf, caplog) -> None:
     conf.update({'config': 'config.json.example'})
     conf.update({'epochs': 1})
     conf.update({'timerange': None})
+    conf.update({'spaces': 'all'})
 
     mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
     mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result)
@@ -173,6 +174,7 @@ def test_fmin_throw_value_error(mocker, default_conf, caplog) -> None:
     conf.update({'config': 'config.json.example'})
     conf.update({'epochs': 1})
     conf.update({'timerange': None})
+    conf.update({'spaces': 'all'})
     mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf)
     mocker.patch('freqtrade.logger.Logger.set_format', MagicMock())
 
@@ -200,6 +202,7 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, default_conf) -> No
     conf.update({'epochs': 1})
     conf.update({'mongodb': False})
     conf.update({'timerange': None})
+    conf.update({'spaces': 'all'})
 
     mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=True)
     mocker.patch('freqtrade.optimize.hyperopt.len', return_value=len(trials.results))
@@ -290,6 +293,7 @@ def test_start_calls_fmin(mocker, default_conf) -> None:
     conf.update({'epochs': 1})
     conf.update({'mongodb': False})
     conf.update({'timerange': None})
+    conf.update({'spaces': 'all'})
 
     hyperopt = Hyperopt(conf)
     hyperopt.trials = trials
@@ -312,6 +316,7 @@ def test_start_uses_mongotrials(mocker, default_conf) -> None:
     conf.update({'epochs': 1})
     conf.update({'mongodb': True})
     conf.update({'timerange': None})
+    conf.update({'spaces': 'all'})
     mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf)
 
     hyperopt = Hyperopt(conf)
@@ -353,3 +358,16 @@ def test_signal_handler(mocker):
     hyperopt = _HYPEROPT
     hyperopt.signal_handler(9, None)
     assert m.call_count == 3
+
+
+def test_has_space():
+    """
+    Test Hyperopt.has_space() method
+    """
+    _HYPEROPT.config.update({'spaces': ['buy', 'roi']})
+    assert _HYPEROPT.has_space('roi')
+    assert _HYPEROPT.has_space('buy')
+    assert not _HYPEROPT.has_space('stoploss')
+
+    _HYPEROPT.config.update({'spaces': ['all']})
+    assert _HYPEROPT.has_space('buy')

freqtrade/tests/test_arguments.py

@@ -118,10 +118,16 @@ def test_parse_args_backtesting_custom() -> None:
 
 
 def test_parse_args_hyperopt_custom() -> None:
-    args = ['-c', 'test_conf.json', 'hyperopt', '--epochs', '20']
+    args = [
+        '-c', 'test_conf.json',
+        'hyperopt',
+        '--epochs', '20',
+        '--spaces', 'buy'
+    ]
     call_args = Arguments(args, '').get_parsed_arg()
     assert call_args.config == 'test_conf.json'
     assert call_args.epochs == 20
     assert call_args.loglevel == logging.INFO
     assert call_args.subparser == 'hyperopt'
+    assert call_args.spaces == 'buy'
     assert call_args.func is not None

freqtrade/tests/test_configuration.py

@@ -261,3 +261,25 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
         'Parameter --export detected: {} ...'.format(config['export']),
         caplog.record_tuples
     )
+
+
+def test_hyperopt_space_argument(mocker, default_conf, caplog) -> None:
+    """
+    Test setup_configuration() function
+    """
+    mocker.patch('freqtrade.configuration.open', mocker.mock_open(
+        read_data=json.dumps(default_conf)
+    ))
+
+    args = [
+        'hyperopt',
+        '--spaces', 'all',
+    ]
+
+    args = Arguments(args, '').get_parsed_arg()
+
+    configuration = Configuration(args)
+    config = configuration.get_config()
+    assert 'spaces' in config
+    assert config['spaces'] is 'all'
+    assert tt.log_has('Parameter -s/--spaces detected: all', caplog.record_tuples)