make --refresh-pairs-cached common option for optimization; added support for it into hyperopt

2024-11-10 10:21:59 +00:00 · 2019-04-22 21:24:45 +03:00 · 2019-04-22 21:24:45 +03:00 · ad85ac3dde
commit ad85ac3dde
parent d3e956f7cc
8 changed files with 205 additions and 74 deletions
--- a/freqtrade/arguments.py
+++ b/freqtrade/arguments.py
@ -141,10 +141,53 @@ class Arguments(object):
            dest='sd_notify',
        )

+    @staticmethod
+    def optimizer_shared_options(parser: argparse.ArgumentParser) -> None:
+        """
+        Parses given common arguments for Backtesting, Edge and Hyperopt modules.
+        :param parser:
+        :return:
+        """
+        parser.add_argument(
+            '-i', '--ticker-interval',
+            help='Specify ticker interval (1m, 5m, 30m, 1h, 1d).',
+            dest='ticker_interval',
+            type=str,
+        )
+        parser.add_argument(
+            '--timerange',
+            help='Specify what timerange of data to use.',
+            default=None,
+            type=str,
+            dest='timerange',
+        )
+        parser.add_argument(
+            '--max_open_trades',
+            help='Specify max_open_trades to use.',
+            default=None,
+            type=int,
+            dest='max_open_trades',
+        )
+        parser.add_argument(
+            '--stake_amount',
+            help='Specify stake_amount.',
+            default=None,
+            type=float,
+            dest='stake_amount',
+        )
+        parser.add_argument(
+            '-r', '--refresh-pairs-cached',
+            help='Refresh the pairs files in tests/testdata with the latest data from the '
+                 'exchange. Use it if you want to run your optimization commands with '
+                 'up-to-date data.',
+            action='store_true',
+            dest='refresh_pairs',
+        )
+
    @staticmethod
    def backtesting_options(parser: argparse.ArgumentParser) -> None:
        """
-        Parses given arguments for Backtesting scripts.
+        Parses given arguments for Backtesting module.
        """
        parser.add_argument(
            '--eps', '--enable-position-stacking',
@ -167,13 +210,6 @@ class Arguments(object):
            action='store_true',
            dest='live',
        )
-        parser.add_argument(
-            '-r', '--refresh-pairs-cached',
-            help='Refresh the pairs files in tests/testdata with the latest data from the '
-                 'exchange. Use it if you want to run your backtesting with up-to-date data.',
-            action='store_true',
-            dest='refresh_pairs',
-        )
        parser.add_argument(
            '--strategy-list',
            help='Provide a commaseparated list of strategies to backtest '
@ -207,15 +243,8 @@ class Arguments(object):
    @staticmethod
    def edge_options(parser: argparse.ArgumentParser) -> None:
        """
-        Parses given arguments for Backtesting scripts.
+        Parses given arguments for Edge module.
        """
-        parser.add_argument(
-            '-r', '--refresh-pairs-cached',
-            help='Refresh the pairs files in tests/testdata with the latest data from the '
-                 'exchange. Use it if you want to run your edge with up-to-date data.',
-            action='store_true',
-            dest='refresh_pairs',
-        )
        parser.add_argument(
            '--stoplosses',
            help='Defines a range of stoploss against which edge will assess the strategy '
@ -225,48 +254,10 @@ class Arguments(object):
            dest='stoploss_range',
        )

-    @staticmethod
-    def optimizer_shared_options(parser: argparse.ArgumentParser) -> None:
-        """
-        Parses given common arguments for Backtesting and Hyperopt scripts.
-        :param parser:
-        :return:
-        """
-        parser.add_argument(
-            '-i', '--ticker-interval',
-            help='Specify ticker interval (1m, 5m, 30m, 1h, 1d).',
-            dest='ticker_interval',
-            type=str,
-        )
-
-        parser.add_argument(
-            '--timerange',
-            help='Specify what timerange of data to use.',
-            default=None,
-            type=str,
-            dest='timerange',
-        )
-
-        parser.add_argument(
-            '--max_open_trades',
-            help='Specify max_open_trades to use.',
-            default=None,
-            type=int,
-            dest='max_open_trades',
-        )
-
-        parser.add_argument(
-            '--stake_amount',
-            help='Specify stake_amount.',
-            default=None,
-            type=float,
-            dest='stake_amount',
-        )
-
    @staticmethod
    def hyperopt_options(parser: argparse.ArgumentParser) -> None:
        """
-        Parses given arguments for Hyperopt scripts.
+        Parses given arguments for Hyperopt module.
        """
        parser.add_argument(
            '--customhyperopt',
--- a/freqtrade/configuration.py
+++ b/freqtrade/configuration.py
@ -324,6 +324,11 @@ class Configuration(object):
            config.update({'print_all': self.args.print_all})
            logger.info('Parameter --print-all detected: %s', config.get('print_all'))

+        # If -r/--refresh-pairs-cached is used we add it to the configuration
+        if 'refresh_pairs' in self.args and self.args.refresh_pairs:
+            config.update({'refresh_pairs': True})
+            logger.info('Parameter -r/--refresh-pairs-cached detected ...')
+
        return config

    def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]:
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@ -116,7 +116,8 @@ def load_pair_history(pair: str,
        return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing)
    else:
        logger.warning('No data for pair: "%s", Interval: %s. '
-                       'Use --refresh-pairs-cached to download the data',
+                       'Use --refresh-pairs-cached option or download_backtest_data.py '
+                       'script to download the data',
                       pair, ticker_interval)
        return None

--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@ -516,6 +516,7 @@ def start(args: Namespace) -> None:
    """
    # Initialize configuration
    config = setup_configuration(args)
+
    logger.info('Starting freqtrade in Backtesting mode')

    # Initialize backtesting object
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@ -20,6 +20,7 @@ from pandas import DataFrame
 from skopt import Optimizer
 from skopt.space import Dimension

+from freqtrade import DependencyException
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
 from freqtrade.data.history import load_data
@ -258,6 +259,8 @@ class Hyperopt(Backtesting):
            datadir=Path(self.config['datadir']) if self.config.get('datadir') else None,
            pairs=self.config['exchange']['pair_whitelist'],
            ticker_interval=self.ticker_interval,
+            refresh_pairs=self.config.get('refresh_pairs', False),
+            exchange=self.exchange,
            timerange=timerange
        )

@ -265,7 +268,10 @@ class Hyperopt(Backtesting):
            self.strategy.advise_indicators = \
                self.custom_hyperopt.populate_indicators  # type: ignore
        dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE)
+
+        # We don't need exchange instance anymore while running hyperopt
        self.exchange = None  # type: ignore
+
        self.load_previous_results()

        cpus = multiprocessing.cpu_count()
@ -295,22 +301,16 @@ class Hyperopt(Backtesting):
        self.log_trials_result()


-def start(args: Namespace) -> None:
+def setup_configuration(args: Namespace) -> Dict[str, Any]:
    """
-    Start Backtesting script
+    Prepare the configuration for the Hyperopt module
    :param args: Cli args from Arguments()
-    :return: None
+    :return: Configuration
    """
-
-    # Remove noisy log messages
-    logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
-
-    # Initialize configuration
-    # Monkey patch the configuration with hyperopt_conf.py
    configuration = Configuration(args, RunMode.HYPEROPT)
-    logger.info('Starting freqtrade in Hyperopt mode')
    config = configuration.load_config()

+    # Ensure we do not use Exchange credentials
    config['exchange']['key'] = ''
    config['exchange']['secret'] = ''

@ -320,7 +320,25 @@ def start(args: Namespace) -> None:
            "Read the documentation at "
            "https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md "
            "to understand how to configure hyperopt.")
-        raise ValueError("--strategy configured but not supported for hyperopt")
+        raise DependencyException("--strategy configured but not supported for hyperopt")
+
+    return config
+
+
+def start(args: Namespace) -> None:
+    """
+    Start Backtesting script
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    # Initialize configuration
+    config = setup_configuration(args)
+
+    logger.info('Starting freqtrade in Hyperopt mode')
+
+    # Remove noisy log messages
+    logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
+
    # Initialize backtesting object
    hyperopt = Hyperopt(config)
    hyperopt.start()
--- a/freqtrade/tests/data/test_history.py
+++ b/freqtrade/tests/data/test_history.py
@ -68,7 +68,10 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None:
    assert ld is None
    assert log_has(
        'No data for pair: "UNITTEST/BTC", Interval: 7m. '
-        'Use --refresh-pairs-cached to download the data', caplog.record_tuples)
+        'Use --refresh-pairs-cached option or download_backtest_data.py '
+        'script to download the data',
+        caplog.record_tuples
+    )


 def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None:
@ -96,9 +99,12 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau
                              refresh_pairs=False,
                              pair='MEME/BTC')
    assert os.path.isfile(file) is False
-    assert log_has('No data for pair: "MEME/BTC", Interval: 1m. '
-                   'Use --refresh-pairs-cached to download the data',
-                   caplog.record_tuples)
+    assert log_has(
+        'No data for pair: "MEME/BTC", Interval: 1m. '
+        'Use --refresh-pairs-cached option or download_backtest_data.py '
+        'script to download the data',
+        caplog.record_tuples
+    )

    # download a new pair if refresh_pairs is set
    history.load_pair_history(datadir=None,
--- a/freqtrade/tests/optimize/test_backtesting.py
+++ b/freqtrade/tests/optimize/test_backtesting.py
@ -260,6 +260,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->

    assert 'refresh_pairs' in config
    assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
+
    assert 'timerange' in config
    assert log_has(
        'Parameter --timerange detected: {} ...'.format(config['timerange']),
--- a/freqtrade/tests/optimize/test_hyperopt.py
+++ b/freqtrade/tests/optimize/test_hyperopt.py
@ -1,16 +1,19 @@
 # pragma pylint: disable=missing-docstring,W0212,C0103
 from datetime import datetime
+import json
 import os
 from unittest.mock import MagicMock

 import pandas as pd
 import pytest

+from freqtrade import DependencyException
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.data.history import load_tickerdata_file
-from freqtrade.optimize.hyperopt import Hyperopt, start
+from freqtrade.optimize.hyperopt import Hyperopt, start, setup_configuration
 from freqtrade.optimize.default_hyperopt import DefaultHyperOpts
 from freqtrade.resolvers import StrategyResolver, HyperOptResolver
+from freqtrade.state import RunMode
 from freqtrade.tests.conftest import log_has, patch_exchange
 from freqtrade.tests.optimize.test_backtesting import get_args

@ -39,6 +42,112 @@ def create_trials(mocker, hyperopt) -> None:
    return [{'loss': 1, 'result': 'foo', 'params': {}}]


+def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, caplog) -> None:
+    mocker.patch('freqtrade.configuration.open', mocker.mock_open(
+        read_data=json.dumps(default_conf)
+    ))
+
+    args = [
+        '--config', 'config.json',
+        'hyperopt'
+    ]
+
+    config = setup_configuration(get_args(args))
+    assert 'max_open_trades' in config
+    assert 'stake_currency' in config
+    assert 'stake_amount' in config
+    assert 'exchange' in config
+    assert 'pair_whitelist' in config['exchange']
+    assert 'datadir' in config
+    assert log_has(
+        'Using data folder: {} ...'.format(config['datadir']),
+        caplog.record_tuples
+    )
+    assert 'ticker_interval' in config
+    assert not log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
+
+    assert 'live' not in config
+    assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples)
+
+    assert 'position_stacking' not in config
+    assert not log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples)
+
+    assert 'refresh_pairs' not in config
+    assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
+
+    assert 'timerange' not in config
+    assert 'runmode' in config
+    assert config['runmode'] == RunMode.HYPEROPT
+
+
+def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplog) -> None:
+    mocker.patch('freqtrade.configuration.open', mocker.mock_open(
+        read_data=json.dumps(default_conf)
+    ))
+    mocker.patch('freqtrade.configuration.Configuration._create_datadir', lambda s, c, x: x)
+
+    args = [
+        '--config', 'config.json',
+        '--datadir', '/foo/bar',
+        'hyperopt',
+        '--ticker-interval', '1m',
+        '--timerange', ':100',
+        '--refresh-pairs-cached',
+        '--enable-position-stacking',
+        '--disable-max-market-positions',
+        '--epochs', '1000',
+        '--spaces', 'all',
+        '--print-all'
+    ]
+
+    config = setup_configuration(get_args(args))
+    assert 'max_open_trades' in config
+    assert 'stake_currency' in config
+    assert 'stake_amount' in config
+    assert 'exchange' in config
+    assert 'pair_whitelist' in config['exchange']
+    assert 'datadir' in config
+    assert config['runmode'] == RunMode.HYPEROPT
+
+    assert log_has(
+        'Using data folder: {} ...'.format(config['datadir']),
+        caplog.record_tuples
+    )
+    assert 'ticker_interval' in config
+    assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
+    assert log_has(
+        'Using ticker_interval: 1m ...',
+        caplog.record_tuples
+    )
+
+    assert 'position_stacking' in config
+    assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples)
+
+    assert 'use_max_market_positions' in config
+    assert log_has('Parameter --disable-max-market-positions detected ...', caplog.record_tuples)
+    assert log_has('max_open_trades set to unlimited ...', caplog.record_tuples)
+
+    assert 'refresh_pairs' in config
+    assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
+
+    assert 'timerange' in config
+    assert log_has(
+        'Parameter --timerange detected: {} ...'.format(config['timerange']),
+        caplog.record_tuples
+    )
+
+    assert 'epochs' in config
+    assert log_has('Parameter --epochs detected ...', caplog.record_tuples)
+
+    assert 'spaces' in config
+    assert log_has(
+        'Parameter -s/--spaces detected: {}'.format(config['spaces']),
+        caplog.record_tuples
+    )
+    assert 'print_all' in config
+    assert log_has('Parameter --print-all detected: True', caplog.record_tuples)
+
+
 def test_hyperoptresolver(mocker, default_conf, caplog) -> None:

    mocker.patch(
@ -72,7 +181,6 @@ def test_start(mocker, default_conf, caplog) -> None:

    args = [
        '--config', 'config.json',
-        '--strategy', 'DefaultStrategy',
        'hyperopt',
        '--epochs', '5'
    ]
@ -107,7 +215,7 @@ def test_start_failure(mocker, default_conf, caplog) -> None:
    ]
    args = get_args(args)
    StrategyResolver({'strategy': 'DefaultStrategy'})
-    with pytest.raises(ValueError):
+    with pytest.raises(DependencyException):
        start(args)
    assert log_has(
        "Please don't use --strategy for hyperopt.",