diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 0ca2f3cc5..988e08029 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -56,8 +56,8 @@ freqtrade -c path/far/far/away/config.json The bot allows you to use multiple configuration files by specifying multiple `-c/--config` configuration options in the command line. Configuration parameters -defined in the last configuration file override parameters with the same name -defined in the previous configuration file specified in the command line. +defined in latter configuration files override parameters with the same name +defined in the previous configuration files specified in the command line earlier. For example, you can make a separate configuration file with your key and secrete for the Exchange you use for trading, specify default configuration file with diff --git a/docs/data-analysis.md b/docs/data-analysis.md index ecd94445b..c89353cc8 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -31,6 +31,16 @@ df = load_trades_from_db("sqlite:///tradesv3.sqlite") df.groupby("pair")["sell_reason"].value_counts() ``` +### Load multiple configuration files + +This option can be usefull to inspect the results of passing in multiple configs in case of problems + +``` python +from freqtrade.configuration import Configuration +config = Configuration.from_files(["config1.json", "config2.json"]) +print(config) +``` + ## Strategy debugging example Debugging a strategy can be time-consuming. FreqTrade offers helper functions to visualize raw data. diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index e564c79ce..237346e37 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -4,7 +4,7 @@ This module contains the configuration class import logging import warnings from argparse import Namespace -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional from freqtrade import OperationalException, constants from freqtrade.configuration.check_exchange import check_exchange @@ -39,43 +39,43 @@ class Configuration(object): return self.config - def _load_config_files(self) -> Dict[str, Any]: + @staticmethod + def from_files(files: List[str]) -> Dict[str, Any]: """ - Iterate through the config files passed in the args, - loading all of them and merging their contents. + Iterate through the config files passed in, loading all of them + and merging their contents. + Files are loaded in sequence, parameters in later configuration files + override the same parameter from an earlier file (last definition wins). + :param files: List of file paths + :return: configuration dictionary """ + # Keep this method as staticmethod, so it can be used from interactive environments config: Dict[str, Any] = {} # We expect here a list of config filenames - for path in self.args.config: - logger.info('Using config: %s ...', path) + for path in files: + logger.info(f'Using config: {path} ...') # Merge config options, overwriting old values config = deep_merge_dicts(load_config_file(path), config) - return config - - def _normalize_config(self, config: Dict[str, Any]) -> None: - """ - Make config more canonical -- i.e. for example add missing parts that we expect - to be normally in it... - """ + # Normalize config if 'internals' not in config: config['internals'] = {} + # validate configuration before returning + logger.info('Validating configuration ...') + validate_config_schema(config) + + return config + def load_config(self) -> Dict[str, Any]: """ Extract information for sys.argv and load the bot configuration :return: Configuration dictionary """ # Load all configs - config: Dict[str, Any] = self._load_config_files() - - # Make resulting config more canonical - self._normalize_config(config) - - logger.info('Validating configuration ...') - validate_config_schema(config) + config: Dict[str, Any] = Configuration.from_files(self.args.config) self._validate_config_consistency(config) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index e05dfc95c..2554982ad 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -14,36 +14,48 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt class DefaultHyperOpts(IHyperOpt): """ Default hyperopt provided by the Freqtrade bot. - You can override it with your own hyperopt + You can override it with your own Hyperopt """ @staticmethod def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Add several indicators needed for buy and sell strategies defined below. + """ + # ADX dataframe['adx'] = ta.ADX(dataframe) + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] + # MFI dataframe['mfi'] = ta.MFI(dataframe) + # RSI dataframe['rsi'] = ta.RSI(dataframe) + # Stochastic Fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] + # Minus-DI dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_upperband'] = bollinger['upper'] + # SAR dataframe['sar'] = ta.SAR(dataframe) + return dataframe @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ - Define the buy strategy parameters to be used by hyperopt + Define the buy strategy parameters to be used by Hyperopt. """ def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Buy strategy Hyperopt will build and use + Buy strategy Hyperopt will build and use. """ conditions = [] + # GUARDS AND TRENDS if 'mfi-enabled' in params and params['mfi-enabled']: conditions.append(dataframe['mfi'] < params['mfi-value']) @@ -79,7 +91,7 @@ class DefaultHyperOpts(IHyperOpt): @staticmethod def indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching strategy parameters + Define your Hyperopt space for searching buy strategy parameters. """ return [ Integer(10, 25, name='mfi-value'), @@ -96,14 +108,14 @@ class DefaultHyperOpts(IHyperOpt): @staticmethod def sell_strategy_generator(params: Dict[str, Any]) -> Callable: """ - Define the sell strategy parameters to be used by hyperopt + Define the sell strategy parameters to be used by Hyperopt. """ def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Sell strategy Hyperopt will build and use + Sell strategy Hyperopt will build and use. """ - # print(params) conditions = [] + # GUARDS AND TRENDS if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: conditions.append(dataframe['mfi'] > params['sell-mfi-value']) @@ -139,7 +151,7 @@ class DefaultHyperOpts(IHyperOpt): @staticmethod def sell_indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching sell strategy parameters + Define your Hyperopt space for searching sell strategy parameters. """ return [ Integer(75, 100, name='sell-mfi-value'), @@ -157,9 +169,9 @@ class DefaultHyperOpts(IHyperOpt): def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file - Only used when --spaces does not include buy + Based on TA indicators. Should be a copy of same method from strategy. + Must align to populate_indicators in this file. + Only used when --spaces does not include buy space. """ dataframe.loc[ ( @@ -174,9 +186,9 @@ class DefaultHyperOpts(IHyperOpt): def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file - Only used when --spaces does not include sell + Based on TA indicators. Should be a copy of same method from strategy. + Must align to populate_indicators in this file. + Only used when --spaces does not include sell space. """ dataframe.loc[ ( @@ -186,4 +198,5 @@ class DefaultHyperOpts(IHyperOpt): (dataframe['fastd'] > 54) ), 'sell'] = 1 + return dataframe diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 55df3acf7..132dd334e 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -133,6 +133,35 @@ def test_load_config_combine_dicts(default_conf, mocker, caplog) -> None: assert log_has('Validating configuration ...', caplog) +def test_from_config(default_conf, mocker, caplog) -> None: + conf1 = deepcopy(default_conf) + conf2 = deepcopy(default_conf) + del conf1['exchange']['key'] + del conf1['exchange']['secret'] + del conf2['exchange']['name'] + conf2['exchange']['pair_whitelist'] += ['NANO/BTC'] + conf2['fiat_display_currency'] = "EUR" + config_files = [conf1, conf2] + + configsmock = MagicMock(side_effect=config_files) + mocker.patch( + 'freqtrade.configuration.configuration.load_config_file', + configsmock + ) + + validated_conf = Configuration.from_files(['test_conf.json', 'test2_conf.json']) + + exchange_conf = default_conf['exchange'] + assert validated_conf['exchange']['name'] == exchange_conf['name'] + assert validated_conf['exchange']['key'] == exchange_conf['key'] + assert validated_conf['exchange']['secret'] == exchange_conf['secret'] + assert validated_conf['exchange']['pair_whitelist'] != conf1['exchange']['pair_whitelist'] + assert validated_conf['exchange']['pair_whitelist'] == conf2['exchange']['pair_whitelist'] + assert validated_conf['fiat_display_currency'] == "EUR" + assert 'internals' in validated_conf + assert log_has('Validating configuration ...', caplog) + + def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) -> None: default_conf['max_open_trades'] = -1 patched_configuration_load_config_file(mocker, default_conf) diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 1a3823afa..fabfdb23e 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -1,11 +1,10 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement from functools import reduce -from math import exp from typing import Any, Callable, Dict, List from datetime import datetime -import numpy as np# noqa F401 +import numpy as np import talib.abstract as ta from pandas import DataFrame from skopt.space import Categorical, Dimension, Integer, Real @@ -16,7 +15,7 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt class SampleHyperOpts(IHyperOpt): """ - This is a sample hyperopt to inspire you. + This is a sample Hyperopt to inspire you. Feel free to customize it. More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md @@ -37,32 +36,44 @@ class SampleHyperOpts(IHyperOpt): """ @staticmethod def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Add several indicators needed for buy and sell strategies defined below. + """ + # ADX dataframe['adx'] = ta.ADX(dataframe) + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] + # MFI dataframe['mfi'] = ta.MFI(dataframe) + # RSI dataframe['rsi'] = ta.RSI(dataframe) + # Stochastic Fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] + # Minus-DI dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_upperband'] = bollinger['upper'] + # SAR dataframe['sar'] = ta.SAR(dataframe) + return dataframe @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ - Define the buy strategy parameters to be used by hyperopt + Define the buy strategy parameters to be used by Hyperopt. """ def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Buy strategy Hyperopt will build and use + Buy strategy Hyperopt will build and use. """ conditions = [] + # GUARDS AND TRENDS if 'mfi-enabled' in params and params['mfi-enabled']: conditions.append(dataframe['mfi'] < params['mfi-value']) @@ -98,7 +109,7 @@ class SampleHyperOpts(IHyperOpt): @staticmethod def indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching strategy parameters + Define your Hyperopt space for searching buy strategy parameters. """ return [ Integer(10, 25, name='mfi-value'), @@ -115,14 +126,14 @@ class SampleHyperOpts(IHyperOpt): @staticmethod def sell_strategy_generator(params: Dict[str, Any]) -> Callable: """ - Define the sell strategy parameters to be used by hyperopt + Define the sell strategy parameters to be used by Hyperopt. """ def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Sell strategy Hyperopt will build and use + Sell strategy Hyperopt will build and use. """ - # print(params) conditions = [] + # GUARDS AND TRENDS if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: conditions.append(dataframe['mfi'] > params['sell-mfi-value']) @@ -158,7 +169,7 @@ class SampleHyperOpts(IHyperOpt): @staticmethod def sell_indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching sell strategy parameters + Define your Hyperopt space for searching sell strategy parameters. """ return [ Integer(75, 100, name='sell-mfi-value'), @@ -176,9 +187,9 @@ class SampleHyperOpts(IHyperOpt): def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file - Only used when --spaces does not include buy + Based on TA indicators. Should be a copy of same method from strategy. + Must align to populate_indicators in this file. + Only used when --spaces does not include buy space. """ dataframe.loc[ ( @@ -193,9 +204,9 @@ class SampleHyperOpts(IHyperOpt): def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file - Only used when --spaces does not include sell + Based on TA indicators. Should be a copy of same method from strategy. + Must align to populate_indicators in this file. + Only used when --spaces does not include sell space. """ dataframe.loc[ ( @@ -205,4 +216,5 @@ class SampleHyperOpts(IHyperOpt): (dataframe['fastd'] > 54) ), 'sell'] = 1 + return dataframe