From 6ef7b7d93d6764ec33d1dbc17678824a0dd4da6e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 2 Mar 2018 21:46:32 +0800 Subject: [PATCH] Complete Backtesting and Hyperopt unit tests --- freqtrade/analyze.py | 2 +- freqtrade/configuration.py | 35 +- freqtrade/logger.py | 5 +- freqtrade/optimize/backtesting.py | 13 +- freqtrade/optimize/hyperopt.py | 1002 +++++++++--------- freqtrade/tests/optimize/test_backtesting.py | 34 +- freqtrade/tests/optimize/test_hyperopt.py | 373 ++++--- freqtrade/tests/optimize/test_optimize.py | 11 - freqtrade/tests/test_analyze.py | 16 +- freqtrade/tests/test_dataframe.py | 8 +- 10 files changed, 851 insertions(+), 648 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index f3126d723..bde0c9f80 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -30,7 +30,7 @@ class Analyze(object): Init Analyze :param config: Bot configuration (use the one from Configuration()) """ - self.logger = Logger(name=__name__).get_logger() + self.logger = Logger(name=__name__, level=config.get('loglevel')).get_logger() self.config = config self.strategy = Strategy(self.config) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 38925bd4e..721def2a4 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -19,7 +19,8 @@ class Configuration(object): """ def __init__(self, args: List[str]) -> None: self.args = args - self.logger = Logger(name=__name__).get_logger() + self.logging = Logger(name=__name__) + self.logger = self.logging.get_logger() self.config = self._load_config() self.show_info() @@ -35,16 +36,24 @@ class Configuration(object): config.update({'strategy': self.args.strategy}) # Add dynamic_whitelist if found - if self.args.dynamic_whitelist: + if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: config.update({'dynamic_whitelist': self.args.dynamic_whitelist}) # Add dry_run_db if found and the bot in dry run if self.args.dry_run_db and config.get('dry_run', False): config.update({'dry_run_db': True}) - # Load Backtesting / Hyperopt + # Log level + if 'loglevel' in self.args and self.args.loglevel: + config.update({'loglevel': self.args.loglevel}) + self.logging.set_level(self.args.loglevel) + + # Load Backtesting config = self._load_backtesting_config(config) + # Load Hyperopt + config = self._load_hyperopt_config(config) + return config def _load_config_file(self, path: str) -> Dict[str, Any]: @@ -64,7 +73,7 @@ class Configuration(object): def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ - Extract information for sys.argv and load Backtesting and Hyperopt configuration + Extract information for sys.argv and load Backtesting configuration :return: configuration as dictionary """ # If -i/--ticker-interval is used we override the configuration parameter @@ -107,6 +116,24 @@ class Configuration(object): return config + def _load_hyperopt_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv and load Hyperopt configuration + :return: configuration as dictionary + """ + # If --realistic-simulation is used we add it to the configuration + if 'epochs' in self.args and self.args.epochs: + config.update({'epochs': self.args.epochs}) + self.logger.info('Parameter --epochs detected ...') + self.logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs')) + + # If --mongodb is used we add it to the configuration + if 'mongodb' in self.args and self.args.mongodb: + config.update({'mongodb': self.args.mongodb}) + self.logger.info('Parameter --use-mongodb detected ...') + + return config + def _validate_config(self, conf: Dict[str, Any]) -> Dict[str, Any]: """ Validate the configuration follow the Config Schema diff --git a/freqtrade/logger.py b/freqtrade/logger.py index 83bbbfd0d..95e55e477 100644 --- a/freqtrade/logger.py +++ b/freqtrade/logger.py @@ -19,9 +19,12 @@ class Logger(object): :return: None """ self.name = name - self.level = level self.logger = None + if level is None: + level = logging.INFO + self.level = level + self._init_logger() def _init_logger(self) -> None: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 60b014872..c15aee1fd 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -5,7 +5,6 @@ This module contains the backtesting logic """ from typing import Dict, Tuple, Any -import logging import arrow from pandas import DataFrame, Series from tabulate import tabulate @@ -20,6 +19,7 @@ from freqtrade.logger import Logger from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade +from memory_profiler import profile class Backtesting(object): """ @@ -30,7 +30,9 @@ class Backtesting(object): backtesting.start() """ def __init__(self, config: Dict[str, Any]) -> None: - self.logging = Logger(name=__name__) + + # Init the logger + self.logging = Logger(name=__name__, level=config['loglevel']) self.logger = self.logging.get_logger() self.config = config @@ -219,6 +221,7 @@ class Backtesting(object): labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] return DataFrame.from_records(trades, columns=labels) + @profile(precision=10) def start(self) -> None: """ Run a backtesting end-to-end @@ -246,10 +249,14 @@ class Backtesting(object): ) max_open_trades = self.config.get('max_open_trades', 0) - preprocessed = self.tickerdata_to_dataframe(data) + # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) + + import pprint + pprint.pprint(min_date) + pprint.pprint(max_date) self.logger.info( 'Measuring data from %s up to %s (%s days)..', min_date.isoformat(), diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8b89e1985..c1d95d6ef 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -1,5 +1,8 @@ -# pragma pylint: disable=missing-docstring,W0212,W0603 +# pragma pylint: disable=too-many-instance-attributes, pointless-string-statement +""" +This module contains the hyperopt logic +""" import json import logging @@ -19,521 +22,574 @@ from hyperopt.mongoexp import MongoTrials from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib -# Monkey patch config -from freqtrade import main # noqa; noqa -from freqtrade import exchange, misc, optimize -from freqtrade.exchange import Bittrex -from freqtrade.misc import load_config -from freqtrade.optimize import backtesting -from freqtrade.optimize.backtesting import backtest -from freqtrade.strategy.strategy import Strategy +from freqtrade.configuration import Configuration +from freqtrade.optimize import load_data +from freqtrade.arguments import Arguments +from freqtrade.optimize.backtesting import Backtesting, setup_configuration +from freqtrade.logger import Logger from user_data.hyperopt_conf import hyperopt_optimize_conf -# Remove noisy log messages -logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) -logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) -logger = logging.getLogger(__name__) - -# set TARGET_TRADES to suit your number concurrent trades so its realistic to the number of days -TARGET_TRADES = 600 -TOTAL_TRIES = 0 -_CURRENT_TRIES = 0 -CURRENT_BEST_LOSS = 100 - -# max average trade duration in minutes -# if eval ends with higher value, we consider it a failed eval -MAX_ACCEPTED_TRADE_DURATION = 300 - -# this is expexted avg profit * expected trade count -# for example 3.5%, 1100 trades, EXPECTED_MAX_PROFIT = 3.85 -# check that the reported Σ% values do not exceed this! -EXPECTED_MAX_PROFIT = 3.0 - -# Configuration and data used by hyperopt -PROCESSED = None # optimize.preprocess(optimize.load_data()) -OPTIMIZE_CONFIG = hyperopt_optimize_conf() - -# Hyperopt Trials -TRIALS_FILE = os.path.join('user_data', 'hyperopt_trials.pickle') -TRIALS = Trials() - -main._CONF = OPTIMIZE_CONFIG - - -def populate_indicators(dataframe: DataFrame) -> DataFrame: +class Hyperopt(Backtesting): """ - Adds several different TA indicators to the given DataFrame + Hyperopt class, this class contains all the logic to run a hyperopt simulation + + To run a backtest: + hyperopt = Hyperopt(config) + hyperopt.start() """ - dataframe['adx'] = ta.ADX(dataframe) - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - dataframe['cci'] = ta.CCI(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['roc'] = ta.ROC(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabolic - dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] + def __init__(self, config: Dict[str, Any]) -> None: - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ + super().__init__(config) - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] - - return dataframe + # Rename the logging to display Hyperopt file instead of Backtesting + self.logging = Logger(name=__name__, level=config['loglevel']) + self.logger = self.logging.get_logger() -def save_trials(trials, trials_path=TRIALS_FILE): - """Save hyperopt trials to file""" - logger.info('Saving Trials to \'{}\''.format(trials_path)) - pickle.dump(trials, open(trials_path, 'wb')) + # set TARGET_TRADES to suit your number concurrent trades so its realistic + # to the number of days + self.target_trades = 600 + self.total_tries = config.get('epochs', 0) + self.current_tries = 0 + self.current_best_loss = 100 + # max average trade duration in minutes + # if eval ends with higher value, we consider it a failed eval + self.max_accepted_trade_duration = 300 -def read_trials(trials_path=TRIALS_FILE): - """Read hyperopt trials file""" - logger.info('Reading Trials from \'{}\''.format(trials_path)) - trials = pickle.load(open(trials_path, 'rb')) - os.remove(trials_path) - return trials + # this is expexted avg profit * expected trade count + # for example 3.5%, 1100 trades, self.expected_max_profit = 3.85 + # check that the reported Σ% values do not exceed this! + self.expected_max_profit = 3.0 + # Configuration and data used by hyperopt + self.processed = None -def log_trials_result(trials): - vals = json.dumps(trials.best_trial['misc']['vals'], indent=4) - results = trials.best_trial['result']['result'] - logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + # Hyperopt Trials + self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') + self.trials = Trials() + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + """ + dataframe['adx'] = ta.ADX(dataframe) + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + dataframe['cci'] = ta.CCI(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['roc'] = ta.ROC(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + # SAR Parabolic + dataframe['sar'] = ta.SAR(dataframe) + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] -def log_results(results): - """ log results if it is better than any previous evaluation """ - global CURRENT_BEST_LOSS + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ - if results['loss'] < CURRENT_BEST_LOSS: - CURRENT_BEST_LOSS = results['loss'] - logger.info('{:5d}/{}: {}. Loss {:.5f}'.format( - results['current_tries'], - results['total_tries'], - results['result'], - results['loss'])) - else: - print('.', end='') - sys.stdout.flush() + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ -def calculate_loss(total_profit: float, trade_count: int, trade_duration: float): - """ objective function, returns smaller number for more optimal results """ - trade_loss = 1 - 0.25 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.8) - profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) - duration_loss = 0.4 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1) - return trade_loss + profit_loss + duration_loss - - -def generate_roi_table(params) -> Dict[str, float]: - roi_table = {} - roi_table["0"] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[str(params['roi_t3'])] = params['roi_p1'] + params['roi_p2'] - roi_table[str(params['roi_t3'] + params['roi_t2'])] = params['roi_p1'] - roi_table[str(params['roi_t3'] + params['roi_t2'] + params['roi_t1'])] = 0 - - return roi_table - - -def roi_space() -> Dict[str, Any]: - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } - - -def stoploss_space() -> Dict[str, Any]: - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } - - -def indicator_space() -> Dict[str, Any]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return { - 'macd_below_zero': hp.choice('macd_below_zero', [ - {'enabled': False}, - {'enabled': True} - ]), - 'mfi': hp.choice('mfi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} - ]), - 'fastd': hp.choice('fastd', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} - ]), - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} - ]), - 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'over_sar': hp.choice('over_sar', [ - {'enabled': False}, - {'enabled': True} - ]), - 'green_candle': hp.choice('green_candle', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_sma': hp.choice('uptrend_sma', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'lower_bb_tema'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema3_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'ht_sine'}, - {'type': 'heiken_reversal_bull'}, - {'type': 'di_cross'}, - ]), - } - - -def hyperopt_space() -> Dict[str, Any]: - return {**indicator_space(), **roi_space(), **stoploss_space()} - - -def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: - conditions.append(dataframe['ema50'] > dataframe['ema100']) - if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: - conditions.append(dataframe['macd'] < 0) - if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: - conditions.append(dataframe['ema5'] > dataframe['ema10']) - if 'mfi' in params and params['mfi']['enabled']: - conditions.append(dataframe['mfi'] < params['mfi']['value']) - if 'fastd' in params and params['fastd']['enabled']: - conditions.append(dataframe['fastd'] < params['fastd']['value']) - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'rsi' in params and params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) - if 'over_sar' in params and params['over_sar']['enabled']: - conditions.append(dataframe['close'] > dataframe['sar']) - if 'green_candle' in params and params['green_candle']['enabled']: - conditions.append(dataframe['close'] > dataframe['open']) - if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: - prevsma = dataframe['sma'].shift(1) - conditions.append(dataframe['sma'] > prevsma) - - # TRIGGERS - triggers = { - 'lower_bb': ( - dataframe['close'] < dataframe['bb_lowerband'] - ), - 'lower_bb_tema': ( - dataframe['tema'] < dataframe['bb_lowerband'] - ), - 'faststoch10': (qtpylib.crossed_above( - dataframe['fastd'], 10.0 - )), - 'ao_cross_zero': (qtpylib.crossed_above( - dataframe['ao'], 0.0 - )), - 'ema3_cross_ema10': (qtpylib.crossed_above( - dataframe['ema3'], dataframe['ema10'] - )), - 'macd_cross_signal': (qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )), - 'sar_reversal': (qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )), - 'ht_sine': (qtpylib.crossed_above( - dataframe['htleadsine'], dataframe['htsine'] - )), - 'heiken_reversal_bull': ( - (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & - (dataframe['ha_low'] == dataframe['ha_open']) - ), - 'di_cross': (qtpylib.crossed_above( - dataframe['plus_di'], dataframe['minus_di'] - )), - } - conditions.append(triggers.get(params['trigger']['type'])) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 + # Chart type + # ------------------------------------ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] return dataframe - return populate_buy_trend + def save_trials(self) -> None: + """ + Save hyperopt trials to file + """ + self.logger.info('Saving Trials to \'%s\'', self.trials_file) + pickle.dump(self.trials, open(self.trials_file, 'wb')) + def read_trials(self) -> Trials: + """ + Read hyperopt trials file + """ + self.logger.info('Reading Trials from \'%s\'', self.trials_file) + trials = pickle.load(open(self.trials_file, 'rb')) + os.remove(self.trials_file) + return trials -def optimizer(params): - global _CURRENT_TRIES + def log_trials_result(self) -> None: + """ + Display Best hyperopt result + """ + vals = json.dumps(self.trials.best_trial['misc']['vals'], indent=4) + results = self.trials.best_trial['result']['result'] + self.logger.info('Best result:\n%s\nwith values:\n%s', results, vals) - if 'roi_t1' in params: - strategy = Strategy() - strategy.minimal_roi = generate_roi_table(params) + def log_results(self, results) -> None: + """ + Log results if it is better than any previous evaluation + """ + if results['loss'] < self.current_best_loss: + self.current_best_loss = results['loss'] + log_msg = '{:5d}/{}: {}. Loss {:.5f}'.format( + results['current_tries'], + results['total_tries'], + results['result'], + results['loss'] + ) + self.logger.info(log_msg) + else: + print('.', end='') + sys.stdout.flush() - backtesting.populate_buy_trend = buy_strategy_generator(params) + def calculate_loss(self, total_profit: float, trade_count: int, trade_duration: float) -> float: + """ + Objective function, returns smaller number for more optimal results + """ + trade_loss = 1 - 0.25 * exp(-(trade_count - self.target_trades) ** 2 / 10 ** 5.8) + profit_loss = max(0, 1 - total_profit / self.expected_max_profit) + duration_loss = 0.4 * min(trade_duration / self.max_accepted_trade_duration, 1) + return trade_loss + profit_loss + duration_loss - results = backtest({'stake_amount': OPTIMIZE_CONFIG['stake_amount'], - 'processed': PROCESSED, - 'stoploss': params['stoploss']}) - result_explanation = format_results(results) + @staticmethod + def generate_roi_table(params) -> Dict[str, float]: + """ + Generate the ROI table thqt will be used by Hyperopt + """ + roi_table = {} + roi_table["0"] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[str(params['roi_t3'])] = params['roi_p1'] + params['roi_p2'] + roi_table[str(params['roi_t3'] + params['roi_t2'])] = params['roi_p1'] + roi_table[str(params['roi_t3'] + params['roi_t2'] + params['roi_t1'])] = 0 - total_profit = results.profit_percent.sum() - trade_count = len(results.index) - trade_duration = results.duration.mean() * 5 + return roi_table - if trade_count == 0 or trade_duration > MAX_ACCEPTED_TRADE_DURATION: - print('.', end='') + @staticmethod + def roi_space() -> Dict[str, Any]: + """ + Values to search for each ROI steps + """ return { - 'status': STATUS_FAIL, - 'loss': float('inf') + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), } - loss = calculate_loss(total_profit, trade_count, trade_duration) + @staticmethod + def stoploss_space() -> Dict[str, Any]: + """ + Stoploss Value to search + """ + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } - _CURRENT_TRIES += 1 + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + } - log_results({ - 'loss': loss, - 'current_tries': _CURRENT_TRIES, - 'total_tries': TOTAL_TRIES, - 'result': result_explanation, - }) + @staticmethod + def hyperopt_space() -> Dict[str, Any]: + """ + Return the space to use during Hyperopt + """ + return { + **Hyperopt.indicator_space(), + **Hyperopt.roi_space(), + **Hyperopt.stoploss_space() + } - return { - 'loss': loss, - 'status': STATUS_OK, - 'result': result_explanation, - } + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) -def format_results(results: DataFrame): - return ('{:6d} trades. Avg profit {: 5.2f}%. ' - 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), - results.profit_percent.sum(), - results.duration.mean() * 5, + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + def optimizer(self, params) -> Dict: + if 'roi_t1' in params: + self.analyze.strategy.minimal_roi = self.generate_roi_table(params) + + self.populate_buy_trend = self.buy_strategy_generator(params) + + results = self.backtest( + { + 'stake_amount': self.config['stake_amount'], + 'processed': self.processed, + 'stoploss': params['stoploss'] + } + ) + result_explanation = self.format_results(results) + + total_profit = results.profit_percent.sum() + trade_count = len(results.index) + trade_duration = results.duration.mean() * 5 + + if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: + print('.', end='') + return { + 'status': STATUS_FAIL, + 'loss': float('inf') + } + + loss = self.calculate_loss(total_profit, trade_count, trade_duration) + + self.current_tries += 1 + + self.log_results( + { + 'loss': loss, + 'current_tries': self.current_tries, + 'total_tries': self.total_tries, + 'result': result_explanation, + } + ) + + return { + 'loss': loss, + 'status': STATUS_OK, + 'result': result_explanation, + } + + @staticmethod + def format_results(results: DataFrame) -> str: + """ + Return the format result in a string + """ + return ('{:6d} trades. Avg profit {: 5.2f}%. ' + 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_BTC.sum(), + results.profit_percent.sum(), + results.duration.mean() * 5, + ) + + def start(self): + timerange = Arguments.parse_timerange(self.config.get('timerange')) + data = load_data( + datadir=self.config.get('datadir'), + pairs=self.config['exchange']['pair_whitelist'], + ticker_interval=self.ticker_interval, + timerange=timerange + ) + + self.analyze.populate_indicators = Hyperopt.populate_indicators + self.processed = self.tickerdata_to_dataframe(data) + + if self.config.get('mongodb'): + self.logger.info('Using mongodb ...') + self.logger.info( + 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' ) + db_name = 'freqtrade_hyperopt' + self.trials = MongoTrials( + arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), + exp_key='exp1' + ) + else: + self.logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, self.signal_handler) + # read trials file if we have one + if os.path.exists(self.trials_file): + self.trials = self.read_trials() -def start(args): - global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES + self.current_tries = len(self.trials.results) + self.total_tries += self.current_tries + self.logger.info( + 'Continuing with trials. Current: {}, Total: {}' + .format(self.current_tries, self.total_tries) + ) - TOTAL_TRIES = args.epochs + try: + # change the Logging format + self.logging.set_format('\n%(message)s') - exchange._API = Bittrex({'key': '', 'secret': ''}) + best_parameters = fmin( + fn=self.optimizer, + space=self.hyperopt_space(), + algo=tpe.suggest, + max_evals=self.total_tries, + trials=self.trials + ) + + results = sorted(self.trials.results, key=itemgetter('loss')) + best_result = results[0]['result'] + + except ValueError: + best_parameters = {} + best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ + 'try with more epochs (param: -e).' + + # Improve best parameter logging display + if best_parameters: + best_parameters = space_eval( + self.hyperopt_space(), + best_parameters + ) + + self.logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) + if 'roi_t1' in best_parameters: + self.logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) + + self.logger.info('Best Result:\n%s', best_result) + + # Store trials result to file to resume next time + self.save_trials() + + def signal_handler(self, sig, frame): + """ + Hyperopt SIGINT handler + """ + self.logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) + + self.save_trials() + self.log_trials_result() + sys.exit(0) + + +def start(args) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + + # Remove noisy log messages + logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) + logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) # Initialize logger - logging.basicConfig( - level=args.loglevel, - format='\n%(message)s', - ) + logger = Logger(name=__name__).get_logger() + logger.info('Starting freqtrade in Hyperopt mode') - logger.info('Using config: %s ...', args.config) - config = load_config(args.config) - pairs = config['exchange']['pair_whitelist'] + # Initialize configuration + #config = setup_configuration(args) - # If -i/--ticker-interval is use we override the configuration parameter - # (that will override the strategy configuration) - if args.ticker_interval: - config.update({'ticker_interval': args.ticker_interval}) + # Monkey patch of the configuration with hyperopt_conf.py + configuration = Configuration(args) + optimize_config = hyperopt_optimize_conf() + config = configuration._load_backtesting_config(optimize_config) + config = configuration._load_hyperopt_config(config) + config['exchange']['key'] = '' + config['exchange']['secret'] = '' - # init the strategy to use - config.update({'strategy': args.strategy}) - strategy = Strategy() - strategy.init(config) - - timerange = misc.parse_timerange(args.timerange) - data = optimize.load_data(args.datadir, pairs=pairs, - ticker_interval=strategy.ticker_interval, - timerange=timerange) - optimize.populate_indicators = populate_indicators - PROCESSED = optimize.tickerdata_to_dataframe(data) - - if args.mongodb: - logger.info('Using mongodb ...') - logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!') - - db_name = 'freqtrade_hyperopt' - TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') - else: - logger.info('Preparing Trials..') - signal.signal(signal.SIGINT, signal_handler) - # read trials file if we have one - if os.path.exists(TRIALS_FILE): - TRIALS = read_trials() - - _CURRENT_TRIES = len(TRIALS.results) - TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES - logger.info( - 'Continuing with trials. Current: {}, Total: {}' - .format(_CURRENT_TRIES, TOTAL_TRIES)) - - try: - best_parameters = fmin( - fn=optimizer, - space=hyperopt_space(), - algo=tpe.suggest, - max_evals=TOTAL_TRIES, - trials=TRIALS - ) - - results = sorted(TRIALS.results, key=itemgetter('loss')) - best_result = results[0]['result'] - - except ValueError: - best_parameters = {} - best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ - 'try with more epochs (param: -e).' - - # Improve best parameter logging display - if best_parameters: - best_parameters = space_eval( - hyperopt_space(), - best_parameters - ) - - logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) - if 'roi_t1' in best_parameters: - logger.info('ROI table:\n%s', generate_roi_table(best_parameters)) - logger.info('Best Result:\n%s', best_result) - - # Store trials result to file to resume next time - save_trials(TRIALS) - - -def signal_handler(sig, frame): - """Hyperopt SIGINT handler""" - logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) - - save_trials(TRIALS) - log_trials_result(TRIALS) - sys.exit(0) + # Initialize backtesting object + hyperopt = Hyperopt(config) + hyperopt.start() diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 7a0158162..d4f705c51 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -5,6 +5,7 @@ import math from typing import List from copy import deepcopy from unittest.mock import MagicMock +from arrow import Arrow import pandas as pd from freqtrade import optimize from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration @@ -255,6 +256,25 @@ def test_backtesting_init(default_conf) -> None: assert callable(backtesting.populate_sell_trend) +def test_tickerdata_to_dataframe(default_conf) -> None: + """ + Test Backtesting.tickerdata_to_dataframe() method + """ + + timerange = ((None, 'line'), None, -100) + tick = optimize.load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) + tickerlist = {'BTC_UNITEST': tick} + + backtesting = _BACKTESTING + data = backtesting.tickerdata_to_dataframe(tickerlist) + assert len(data['BTC_UNITEST']) == 100 + + # Load Analyze to compare the result between Backtesting function and Analyze are the same + analyze = Analyze(default_conf) + data2 = analyze.tickerdata_to_dataframe(tickerlist) + assert data['BTC_UNITEST'].equals(data2['BTC_UNITEST']) + + def test_get_timeframe() -> None: """ Test Backtesting.get_timeframe() method @@ -308,8 +328,18 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: """ Test Backtesting.start() method """ - mocker.patch.multiple('freqtrade.optimize', load_data=mocked_load_data) - mocker.patch('freqtrade.exchange.get_ticker_history', MagicMock) + def get_timeframe(input1, input2): + return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) + + mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) + mocker.patch('freqtrade.optimize.load_data', mocked_load_data) + mocker.patch('freqtrade.exchange.get_ticker_history') + mocker.patch.multiple( + 'freqtrade.optimize.backtesting.Backtesting', + backtest=MagicMock(), + _generate_text_table=MagicMock(return_value='1'), + get_timeframe=get_timeframe, + ) conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'] = ['BTC_UNITEST'] diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index f127ac8fd..63e4404ff 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,117 +1,108 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 import logging +import os +import pytest +from copy import deepcopy -from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ - log_results, save_trials, read_trials, generate_roi_table +#from freqtrade.optimize.hyperopt import EXPECTED_MAX_PROFIT, start, \ +# log_results, save_trials, read_trials, generate_roi_table +from unittest.mock import MagicMock + +from freqtrade.optimize.hyperopt import Hyperopt, start +import freqtrade.tests.conftest as tt # test tools -def test_loss_calculation_prefer_correct_trade_count(): - correct = calculate_loss(1, TARGET_TRADES, 20) - over = calculate_loss(1, TARGET_TRADES + 100, 20) - under = calculate_loss(1, TARGET_TRADES - 100, 20) - assert over > correct - assert under > correct +# Avoid to reinit the same object again and again +_HYPEROPT = Hyperopt(tt.default_conf()) -def test_loss_calculation_prefer_shorter_trades(): - shorter = calculate_loss(1, 100, 20) - longer = calculate_loss(1, 100, 30) - assert shorter < longer - - -def test_loss_calculation_has_limited_profit(): - correct = calculate_loss(EXPECTED_MAX_PROFIT, TARGET_TRADES, 20) - over = calculate_loss(EXPECTED_MAX_PROFIT * 2, TARGET_TRADES, 20) - under = calculate_loss(EXPECTED_MAX_PROFIT / 2, TARGET_TRADES, 20) - assert over == correct - assert under > correct - - -def create_trials(mocker): +# Functions for recurrent object patching +def create_trials(mocker) -> None: """ When creating trials, mock the hyperopt Trials so that *by default* - we don't create any pickle'd files in the filesystem - we might have a pickle'd file so make sure that we return false when looking for it """ - mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', - return_value='freqtrade/tests/optimize/ut_trials.pickle') - mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', - return_value=False) - mocker.patch('freqtrade.optimize.hyperopt.save_trials', - return_value=None) - mocker.patch('freqtrade.optimize.hyperopt.read_trials', - return_value=None) - mocker.patch('freqtrade.optimize.hyperopt.os.remove', - return_value=True) + _HYPEROPT.trials_file = os.path.join('freqtrade', 'tests', 'optimize','ut_trials.pickle') + + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False) + mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + return mocker.Mock( - results=[{ - 'loss': 1, - 'result': 'foo', - 'status': 'ok' - }], + results=[ + { + 'loss': 1, + 'result': 'foo', + 'status': 'ok' + } + ], best_trial={'misc': {'vals': {'adx': 999}}} ) -def test_start_calls_fmin(mocker): - trials = create_trials(mocker) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials) - mocker.patch('freqtrade.optimize.hyperopt.sorted', - return_value=trials.results) - mocker.patch('freqtrade.optimize.preprocess') - mocker.patch('freqtrade.optimize.load_data') - mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) +# Unit tests +def test_loss_calculation_prefer_correct_trade_count() -> None: + """ + Test Hyperopt.calculate_loss() + """ + hyperopt = _HYPEROPT - args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False, - timerange=None) - start(args) - - mock_fmin.assert_called_once() + correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20) + over = hyperopt.calculate_loss(1, hyperopt.target_trades + 100, 20) + under = hyperopt.calculate_loss(1, hyperopt.target_trades - 100, 20) + assert over > correct + assert under > correct -def test_start_uses_mongotrials(mocker): - mock_mongotrials = mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', - return_value=create_trials(mocker)) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.load_data') - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) +def test_loss_calculation_prefer_shorter_trades() -> None: + """ + Test Hyperopt.calculate_loss() + """ + hyperopt = _HYPEROPT - args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True, - timerange=None) - start(args) - - mock_mongotrials.assert_called_once() + shorter = hyperopt.calculate_loss(1, 100, 20) + longer = hyperopt.calculate_loss(1, 100, 30) + assert shorter < longer -def test_log_results_if_loss_improves(mocker): - logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info') - global CURRENT_BEST_LOSS - CURRENT_BEST_LOSS = 2 - log_results({ - 'loss': 1, - 'current_tries': 1, - 'total_tries': 2, - 'result': 'foo' - }) +def test_loss_calculation_has_limited_profit() -> None: + hyperopt = _HYPEROPT - logger.assert_called_once() + correct = hyperopt.calculate_loss(hyperopt.expected_max_profit, hyperopt.target_trades, 20) + over = hyperopt.calculate_loss(hyperopt.expected_max_profit * 2, hyperopt.target_trades, 20) + under = hyperopt.calculate_loss(hyperopt.expected_max_profit / 2, hyperopt.target_trades, 20) + assert over == correct + assert under > correct -def test_no_log_if_loss_does_not_improve(mocker): - logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info') - global CURRENT_BEST_LOSS - CURRENT_BEST_LOSS = 2 - log_results({ - 'loss': 3, - }) - - assert not logger.called +def test_log_results_if_loss_improves(caplog) -> None: + hyperopt = _HYPEROPT + hyperopt.current_best_loss = 2 + hyperopt.log_results( + { + 'loss': 1, + 'current_tries': 1, + 'total_tries': 2, + 'result': 'foo' + } + ) + assert tt.log_has(' 1/2: foo. Loss 1.00000', caplog.record_tuples) -def test_fmin_best_results(mocker, caplog): - caplog.set_level(logging.INFO) +def test_no_log_if_loss_does_not_improve(caplog) -> None: + hyperopt = _HYPEROPT + hyperopt.current_best_loss = 2 + hyperopt.log_results( + { + 'loss': 3, + } + ) + assert caplog.record_tuples == [] + + +def test_fmin_best_results(mocker, default_conf, caplog) -> None: fmin_result = { "macd_below_zero": 0, "adx": 1, @@ -136,38 +127,65 @@ def test_fmin_best_results(mocker, caplog): "roi_p3": 3, } - mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', return_value=create_trials(mocker)) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.load_data') - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'timerange': None}) - args = mocker.Mock(epochs=1, config='config.json.example', - timerange=None) - start(args) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + + hyperopt = Hyperopt(conf) + hyperopt.trials = create_trials(mocker) + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() exists = [ - 'Best parameters', + 'Best parameters:', '"adx": {\n "enabled": true,\n "value": 15.0\n },', + '"fastd": {\n "enabled": true,\n "value": 40.0\n },', '"green_candle": {\n "enabled": true\n },', + '"macd_below_zero": {\n "enabled": false\n },', '"mfi": {\n "enabled": false\n },', + '"over_sar": {\n "enabled": false\n },', + '"roi_p1": 1.0,', + '"roi_p2": 2.0,', + '"roi_p3": 3.0,', + '"roi_t1": 1.0,', + '"roi_t2": 2.0,', + '"roi_t3": 3.0,', + '"rsi": {\n "enabled": true,\n "value": 37.0\n },', + '"stoploss": -0.1,', '"trigger": {\n "type": "faststoch10"\n },', - '"stoploss": -0.1', + '"uptrend_long_ema": {\n "enabled": true\n },', + '"uptrend_short_ema": {\n "enabled": false\n },', + '"uptrend_sma": {\n "enabled": false\n }', + 'ROI table:\n{\'0\': 6.0, \'3.0\': 3.0, \'5.0\': 1.0, \'6.0\': 0}', + 'Best Result:\nfoo' ] - for line in exists: assert line in caplog.text -def test_fmin_throw_value_error(mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', return_value=create_trials(mocker)) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.load_data') +def test_fmin_throw_value_error(mocker, default_conf, caplog) -> None: + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError()) - args = mocker.Mock(epochs=1, config='config.json.example', - timerange=None) - start(args) + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'timerange': None}) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + + hyperopt = Hyperopt(conf) + hyperopt.trials = create_trials(mocker) + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() exists = [ 'Best Result:', @@ -179,68 +197,80 @@ def test_fmin_throw_value_error(mocker, caplog): assert line in caplog.text -def test_resuming_previous_hyperopt_results_succeeds(mocker): - import freqtrade.optimize.hyperopt as hyperopt +def test_resuming_previous_hyperopt_results_succeeds(mocker, default_conf) -> None: trials = create_trials(mocker) - mocker.patch('freqtrade.optimize.hyperopt.TRIALS', - return_value=trials) - mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', - return_value=True) - mocker.patch('freqtrade.optimize.hyperopt.len', - return_value=len(trials.results)) - mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials', - return_value=trials) - mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials', - return_value=None) - mocker.patch('freqtrade.optimize.hyperopt.sorted', - return_value=trials.results) - mocker.patch('freqtrade.optimize.preprocess') - mocker.patch('freqtrade.optimize.load_data') - mocker.patch('freqtrade.optimize.hyperopt.fmin', - return_value={}) - args = mocker.Mock(epochs=1, - config='config.json.example', - mongodb=False, - timerange=None) - start(args) + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'mongodb': False}) + conf.update({'timerange': None}) + + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.len', return_value=len(trials.results)) + mock_read = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.read_trials', + return_value=trials + ) + mock_save = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.save_trials', + return_value=None + ) + mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + + hyperopt = Hyperopt(conf) + hyperopt.trials = trials + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() mock_read.assert_called_once() mock_save.assert_called_once() - current_tries = hyperopt._CURRENT_TRIES - total_tries = hyperopt.TOTAL_TRIES + current_tries = hyperopt.current_tries + total_tries = hyperopt.total_tries assert current_tries == len(trials.results) assert total_tries == (current_tries + len(trials.results)) -def test_save_trials_saves_trials(mocker): +def test_save_trials_saves_trials(mocker, caplog) -> None: + create_trials(mocker) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + + hyperopt = _HYPEROPT + mocker.patch('freqtrade.optimize.hyperopt.open', return_value=hyperopt.trials_file) + + hyperopt.save_trials() + + assert tt.log_has( + 'Saving Trials to \'freqtrade/tests/optimize/ut_trials.pickle\'', + caplog.record_tuples + ) + mock_dump.assert_called_once() + + +def test_read_trials_returns_trials_file(mocker, default_conf, caplog) -> None: trials = create_trials(mocker) - mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', - return_value=None) - trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', - return_value='ut_trials.pickle') - mocker.patch('freqtrade.optimize.hyperopt.open', - return_value=trials_path) - save_trials(trials, trials_path) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', return_value=trials) + mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', return_value=mock_load) - mock_dump.assert_called_once_with(trials, trials_path) - - -def test_read_trials_returns_trials_file(mocker): - trials = create_trials(mocker) - mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', - return_value=trials) - mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', - return_value=mock_load) - - assert read_trials() == trials + hyperopt = _HYPEROPT + hyperopt_trial = hyperopt.read_trials() + assert tt.log_has( + 'Reading Trials from \'freqtrade/tests/optimize/ut_trials.pickle\'', + caplog.record_tuples + ) + assert hyperopt_trial == trials mock_open.assert_called_once() mock_load.assert_called_once() -def test_roi_table_generation(): +def test_roi_table_generation() -> None: params = { 'roi_t1': 5, 'roi_t2': 10, @@ -249,4 +279,49 @@ def test_roi_table_generation(): 'roi_p2': 2, 'roi_p3': 3, } - assert generate_roi_table(params) == {'0': 6, '15': 3, '25': 1, '30': 0} + + hyperopt = _HYPEROPT + assert hyperopt.generate_roi_table(params) == {'0': 6, '15': 3, '25': 1, '30': 0} + + +def test_start_calls_fmin(mocker, default_conf) -> None: + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'mongodb': False}) + conf.update({'timerange': None}) + + hyperopt = Hyperopt(conf) + hyperopt.trials = trials + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() + mock_fmin.assert_called_once() + + +def test_start_uses_mongotrials(mocker, default_conf) -> None: + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + mock_mongotrials = mocker.patch( + 'freqtrade.optimize.hyperopt.MongoTrials', + return_value=create_trials(mocker) + ) + + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'mongodb': True}) + conf.update({'timerange': None}) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + + hyperopt = Hyperopt(conf) + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() + mock_mongotrials.assert_called_once() + mock_fmin.assert_called_once() diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index b1459198d..4e35507f5 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -6,7 +6,6 @@ import logging import uuid from shutil import copyfile from freqtrade import optimize -from freqtrade.analyze import Analyze from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ download_backtesting_testdata, load_tickerdata_file, trim_tickerlist from freqtrade.misc import file_dump_json @@ -220,16 +219,6 @@ def test_init(default_conf, mocker) -> None: ) -def test_tickerdata_to_dataframe(default_conf) -> None: - analyze = Analyze(default_conf) - - timerange = ((None, 'line'), None, -100) - tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) - tickerlist = {'BTC_UNITEST': tick} - data = analyze.tickerdata_to_dataframe(tickerlist) - assert len(data['BTC_UNITEST']) == 100 - - def test_trim_tickerlist() -> None: with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: ticker_list = json.load(data_file) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 3c1687ab3..7ada49028 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -10,8 +10,9 @@ import logging import arrow from pandas import DataFrame -import freqtrade.tests.conftest as tt # test tools from freqtrade.analyze import Analyze, SignalType +from freqtrade.optimize.__init__ import load_tickerdata_file +import freqtrade.tests.conftest as tt # test tools # Avoid to reinit the same object again and again @@ -173,3 +174,16 @@ def test_parse_ticker_dataframe(ticker_history, ticker_history_without_bv): # Test file without BV data dataframe = Analyze.parse_ticker_dataframe(ticker_history_without_bv) assert dataframe.columns.tolist() == columns + + +def test_tickerdata_to_dataframe(default_conf) -> None: + """ + Test Analyze.tickerdata_to_dataframe() method + """ + analyze = Analyze(default_conf) + + timerange = ((None, 'line'), None, -100) + tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) + tickerlist = {'BTC_UNITEST': tick} + data = analyze.tickerdata_to_dataframe(tickerlist) + assert len(data['BTC_UNITEST']) == 100 diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index 9af42a30e..fcec2d119 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -1,17 +1,19 @@ # pragma pylint: disable=missing-docstring, C0103 import pandas -import freqtrade.optimize -from freqtrade import analyze +from freqtrade.optimize import load_data +from freqtrade.analyze import Analyze, SignalType _pairs = ['BTC_ETH'] def load_dataframe_pair(pairs): - ld = freqtrade.optimize.load_data(None, ticker_interval=5, pairs=pairs) + ld = load_data(None, ticker_interval=5, pairs=pairs) assert isinstance(ld, dict) assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] + + analyze = Analyze({'strategy': 'default_strategy'}) dataframe = analyze.analyze_ticker(dataframe) return dataframe