From 2496aa8e3f027cfb95a6628204442ca464739121 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 10:59:26 +0100 Subject: [PATCH 001/317] Add convert-data template subcommands --- freqtrade/configuration/arguments.py | 13 ++++++++++++- freqtrade/utils.py | 8 ++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index b2197619d..396b55ef5 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -47,6 +47,8 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] +ARGS_CONVERT_DATA = [] + ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", "timeframes", "erase"] @@ -131,7 +133,7 @@ class Arguments: self._build_args(optionlist=['version'], parser=self.parser) from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge - from freqtrade.utils import (start_create_userdir, start_download_data, + from freqtrade.utils import (start_create_userdir, start_convert_data, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, start_list_strategies, start_new_hyperopt, @@ -251,6 +253,15 @@ class Arguments: download_data_cmd.set_defaults(func=start_download_data) self._build_args(optionlist=ARGS_DOWNLOAD_DATA, parser=download_data_cmd) + # Add convert-data subcommand + convert_data_cmd = subparsers.add_parser( + 'convert-data', + help='Convert data from one format to another.', + parents=[_common_parser], + ) + convert_data_cmd.set_defaults(func=start_convert_data) + self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_data_cmd) + # Add Plotting subcommand plot_dataframe_cmd = subparsers.add_parser( 'plot-dataframe', diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 5a5662e4b..59a8dcbb3 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -241,6 +241,14 @@ def start_list_strategies(args: Dict[str, Any]) -> None: print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) +def start_convert_data(args: Dict[str, Any]) -> None: + """ + Convert data from one format to another + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + print(config) + + def start_list_timeframes(args: Dict[str, Any]) -> None: """ Print ticker intervals (timeframes) available on Exchange From e5a61667ddf60061a5e5477fa3b97da9a6563a9a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 14:56:48 +0100 Subject: [PATCH 002/317] Implement first version of jsondatahandler --- freqtrade/configuration/arguments.py | 6 +- freqtrade/data/datahandlers/__init__.py | 20 ++++ freqtrade/data/datahandlers/idatahandler.py | 97 ++++++++++++++++ .../data/datahandlers/jsondatahandler.py | 105 ++++++++++++++++++ 4 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 freqtrade/data/datahandlers/__init__.py create mode 100644 freqtrade/data/datahandlers/idatahandler.py create mode 100644 freqtrade/data/datahandlers/jsondatahandler.py diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 396b55ef5..44bc71038 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -65,9 +65,9 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "print_c ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", "print_json", "hyperopt_show_no_header"] -NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs", - "list-strategies", "hyperopt-list", "hyperopt-show", "plot-dataframe", - "plot-profit"] +NO_CONF_REQURIED = ["convert-data", "download-data", "list-timeframes", "list-markets", + "list-pairs", "list-strategies", "hyperopt-list", "hyperopt-show", + "plot-dataframe", "plot-profit"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] diff --git a/freqtrade/data/datahandlers/__init__.py b/freqtrade/data/datahandlers/__init__.py new file mode 100644 index 000000000..b3aa2e3a4 --- /dev/null +++ b/freqtrade/data/datahandlers/__init__.py @@ -0,0 +1,20 @@ +from .idatahandler import IDataHandler + + +def get_datahandlerclass(datatype: str) -> IDataHandler: + """ + Get datahandler class. + Could be done using Resolvers, but since this may be called often and resolvers + are rather expensive, doing this directly should improve performance. + :param datatype: datatype to use. + :return: Datahandler class + """ + + if datatype == 'json': + from .jsondatahandler import JsonDataHandler + return JsonDataHandler + elif datatype == 'jsongz': + from .jsondatahandler import JsonGzDataHandler + return JsonGzDataHandler + else: + raise ValueError(f"No datahandler for datatype {datatype} available.") diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/datahandlers/idatahandler.py new file mode 100644 index 000000000..ffe50b14e --- /dev/null +++ b/freqtrade/data/datahandlers/idatahandler.py @@ -0,0 +1,97 @@ +""" +Abstract datahandler interface. +It's subclasses handle and storing data from disk. + +""" + +from abc import ABC, abstractmethod, abstractclassmethod +from pathlib import Path +from typing import Dict, List, Optional + +from pandas import DataFrame + +from freqtrade.configuration import TimeRange + + +class IDataHandler(ABC): + + def __init__(self, datadir: Path, pair: str) -> None: + self._datadir = datadir + self._pair = pair + + @abstractclassmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs available in this datadir + """ + + @abstractmethod + def ohlcv_store(self, timeframe: str, data: DataFrame): + """ + Store data + """ + + @abstractmethod + def ohlcv_append(self, timeframe: str, data: DataFrame): + """ + Append data to existing files + """ + + @abstractmethod + def ohlcv_load(self, timeframe: str, timerange: Optional[TimeRange] = None) -> DataFrame: + """ + Load data for one pair + :return: Dataframe + """ + + @abstractclassmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs available in this datadir + """ + + @abstractmethod + def trades_store(self, data: DataFrame): + """ + Store data + """ + + @abstractmethod + def trades_append(self, data: DataFrame): + """ + Append data to existing files + """ + + @abstractmethod + def trades_load(self, timerange: Optional[TimeRange] = None): + """ + Load data for one pair + :return: Dataframe + """ + + @staticmethod + def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: + """ + TODO: investigate if this is needed ... we can probably cover this in a dataframe + Trim tickerlist based on given timerange + """ + if not tickerlist: + return tickerlist + + start_index = 0 + stop_index = len(tickerlist) + + if timerange.starttype == 'date': + while (start_index < len(tickerlist) and + tickerlist[start_index][0] < timerange.startts * 1000): + start_index += 1 + + if timerange.stoptype == 'date': + while (stop_index > 0 and + tickerlist[stop_index-1][0] > timerange.stopts * 1000): + stop_index -= 1 + + if start_index > stop_index: + raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') + + return tickerlist[start_index:stop_index] diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py new file mode 100644 index 000000000..214958251 --- /dev/null +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -0,0 +1,105 @@ +import re +from pathlib import Path +from typing import Dict, List, Optional + +from pandas import DataFrame + +from freqtrade import misc +from freqtrade.configuration import TimeRange + +from .idatahandler import IDataHandler + + +class JsonDataHandler(IDataHandler): + + _use_zip = False + + @classmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs available in this datadir + """ + return [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name)[0].replace('_', ' /') + for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] + + def ohlcv_store(self, timeframe: str, data: DataFrame): + """ + Store data + """ + raise NotImplementedError() + + def ohlcv_append(self, timeframe: str, data: DataFrame): + """ + Append data to existing files + """ + raise NotImplementedError() + + def ohlcv_load(self, timeframe: str, timerange: Optional[TimeRange] = None) -> DataFrame: + """ + Load data for one pair + :return: Dataframe + """ + filename = JsonDataHandler._pair_data_filename(self.datadir, self._pair, + self._pair, timeframe) + pairdata = misc.file_load_json(filename) + if not pairdata: + return [] + + if timerange: + pairdata = IDataHandler.trim_tickerlist(pairdata, timerange) + return pairdata + + @classmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs available in this datadir + """ + return [re.search(r'^(\S+)(?=\-trades.json)', p.name)[0].replace('_', '/') + for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] + + def trades_store(self, data: List[Dict]): + """ + Store data + """ + filename = self._pair_trades_filename(self._datadir, self._pair) + misc.file_dump_json(filename, data, is_zip=self._use_zip) + + def trades_append(self, data: DataFrame): + """ + Append data to existing files + """ + raise NotImplementedError() + + def trades_load(self, timerange: Optional[TimeRange] = None) -> List[Dict]: + """ + Load a pair from file, either .json.gz or .json + # TODO: validate timerange ... + :return: List of trades + """ + filename = self._pair_trades_filename(self._datadir, self._pair) + tradesdata = misc.file_load_json(filename) + if not tradesdata: + return [] + + return tradesdata + + @classmethod + def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + pair_s = pair.replace("/", "_") + filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') + return filename + + @classmethod + def _get_file_extension(cls): + return "json.gz" if cls._use_zip else "json" + + @classmethod + def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + pair_s = pair.replace("/", "_") + filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') + return filename + + +class JsonGzDataHandler(JsonDataHandler): + + _use_zip = True From cd4466a62655dbec5c0e811af87fd530b3ea0b28 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 15:04:45 +0100 Subject: [PATCH 003/317] Add convert_* methods --- freqtrade/utils.py | 50 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 59a8dcbb3..82d781fe5 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -17,6 +17,7 @@ from freqtrade.configuration import (Configuration, TimeRange, from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY +from freqtrade.data.datahandlers import get_datahandlerclass from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) @@ -241,12 +242,53 @@ def start_list_strategies(args: Dict[str, Any]) -> None: print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) +def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str): + """ + TODO: move this to converter.py (?) + """ + SrcClass = get_datahandlerclass(convert_from) + TrgClass = get_datahandlerclass(convert_to) + + if 'pairs' not in config: + config['pairs'] = SrcClass.trades_get_pairs(config['datadir']) + logger.info(f"Converting trades for {config['pairs']}") + + for pair in config['pairs']: + src = SrcClass(config['datadir'], pair) + trg = TrgClass(config['datadir'], pair) + data = src.trades_load() + logger.info(f"Converting {len(data)} trades for {pair}") + trg.trades_store(data) + + +def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str): + """ + TODO: move this to converter.py (?) + """ + SrcClass = get_datahandlerclass(convert_from) + TrgClass = get_datahandlerclass(convert_to) + + if 'pairs' not in config: + config['pairs'] = SrcClass.ohclv_get_pairs(config['datadir'], config['ticker_interval']) + logger.info(f"Converting OHLCV for {config['pairs']}") + + for pair in config['pairs']: + src = SrcClass(config['datadir'], pair) + trg = TrgClass(config['datadir'], pair) + data = src.ohlcv_load() + logger.info(f"Converting {len(data)} candles for {pair}") + trg.ohlcv_store(data) + + def start_convert_data(args: Dict[str, Any]) -> None: """ Convert data from one format to another """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - print(config) + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + from pprint import pprint + pprint(config) + + # convert_trades_format(config, 'json', 'jsongz') def start_list_timeframes(args: Dict[str, Any]) -> None: @@ -452,10 +494,10 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: n = config.get('hyperopt_show_index', -1) if n > trials_epochs: raise OperationalException( - f"The index of the epoch to show should be less than {trials_epochs + 1}.") + f"The index of the epoch to show should be less than {trials_epochs + 1}.") if n < -trials_epochs: raise OperationalException( - f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") + f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") # Translate epoch index from human-readable format to pythonic if n > 0: From c3064dfd2b56e8d74ef8c07219b0e119cd0626c6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2019 15:06:33 +0100 Subject: [PATCH 004/317] Enhance validation constants --- freqtrade/constants.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d7c6249d5..2f60bef2d 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -18,6 +18,7 @@ REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'PriceFilter'] +AVAILABLE_DATAHANDLERS = ['json', 'jsongz'] DRY_RUN_WALLET = 1000 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons @@ -213,6 +214,16 @@ CONF_SCHEMA = { 'process_throttle_secs': {'type': 'integer'}, 'interval': {'type': 'integer'}, 'sd_notify': {'type': 'boolean'}, + 'dataformat_ohlcv': { + 'type': 'string', + 'enum': AVAILABLE_DATAHANDLERS, + 'default': 'json' + }, + 'dataformat_trades': { + 'type': 'string', + 'enum': AVAILABLE_DATAHANDLERS, + 'default': 'jsongz' + } } } }, @@ -280,5 +291,6 @@ CONF_SCHEMA = { 'unfilledtimeout', 'stoploss', 'minimal_roi', + 'internals', ] } From 2a6b542b098f4ada5fd75149437e9dcb14474450 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 09:59:01 +0100 Subject: [PATCH 005/317] Add second subcommand to allow conversation of ohlcv and trades data seprately --- freqtrade/configuration/arguments.py | 20 +++++++++++++++----- freqtrade/configuration/cli_options.py | 12 ++++++++++++ freqtrade/utils.py | 10 +++++++--- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 44bc71038..a20ecaa43 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -47,7 +47,7 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] -ARGS_CONVERT_DATA = [] +ARGS_CONVERT_DATA = ["format_from", "format_to"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", "timeframes", "erase"] @@ -65,8 +65,9 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "print_c ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", "print_json", "hyperopt_show_no_header"] -NO_CONF_REQURIED = ["convert-data", "download-data", "list-timeframes", "list-markets", - "list-pairs", "list-strategies", "hyperopt-list", "hyperopt-show", +NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", + "list-timeframes", "list-markets", "list-pairs", + "list-strategies", "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] @@ -256,10 +257,19 @@ class Arguments: # Add convert-data subcommand convert_data_cmd = subparsers.add_parser( 'convert-data', - help='Convert data from one format to another.', + help='Convert OHLCV data from one format to another.', parents=[_common_parser], ) - convert_data_cmd.set_defaults(func=start_convert_data) + convert_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=True)) + self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_data_cmd) + + # Add convert-data subcommand + convert_data_cmd = subparsers.add_parser( + 'convert-trade-data', + help='Convert trade-data from one format to another.', + parents=[_common_parser], + ) + convert_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False)) self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_data_cmd) # Add Plotting subcommand diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index 4b6429f20..ffaf9ea66 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -332,6 +332,18 @@ AVAILABLE_CLI_OPTIONS = { 'desired timeframe as specified as --timeframes/-t.', action='store_true', ), + "format_from": Arg( + '--format-from', + help='Source format for data conversation.', + choices=constants.AVAILABLE_DATAHANDLERS, + required=True, + ), + "format_to": Arg( + '--format-to', + help='Destination format for data conversation.', + choices=constants.AVAILABLE_DATAHANDLERS, + required=True, + ), "exchange": Arg( '--exchange', help=f'Exchange name (default: `{constants.DEFAULT_EXCHANGE}`). ' diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 82d781fe5..d02c80f7f 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -280,15 +280,19 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: trg.ohlcv_store(data) -def start_convert_data(args: Dict[str, Any]) -> None: +def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ Convert data from one format to another """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) from pprint import pprint pprint(config) - - # convert_trades_format(config, 'json', 'jsongz') + if ohlcv: + convert_ohlcv_format(config, + convert_from=args['format_from'], convert_to=args['format_to']) + else: + convert_trades_format(config, + convert_from=args['format_from'], convert_to=args['format_to']) def start_list_timeframes(args: Dict[str, Any]) -> None: From f8b8b9ac631dfe36083a3f0f7c1ccc086a81dad0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 10:01:39 +0100 Subject: [PATCH 006/317] Convert to Path temporarily --- freqtrade/utils.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index d02c80f7f..4661e84b2 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -245,17 +245,18 @@ def start_list_strategies(args: Dict[str, Any]) -> None: def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str): """ TODO: move this to converter.py (?) + TODO: remove Path conversation once PR is merged and this is rebased """ SrcClass = get_datahandlerclass(convert_from) TrgClass = get_datahandlerclass(convert_to) if 'pairs' not in config: - config['pairs'] = SrcClass.trades_get_pairs(config['datadir']) + config['pairs'] = SrcClass.trades_get_pairs(Path(config['datadir'])) logger.info(f"Converting trades for {config['pairs']}") for pair in config['pairs']: - src = SrcClass(config['datadir'], pair) - trg = TrgClass(config['datadir'], pair) + src = SrcClass(Path(config['datadir'], pair)) + trg = TrgClass(Path(config['datadir'], pair)) data = src.trades_load() logger.info(f"Converting {len(data)} trades for {pair}") trg.trades_store(data) @@ -264,17 +265,18 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str): """ TODO: move this to converter.py (?) + TODO: remove Path conversation once PR is merged and this is rebased """ SrcClass = get_datahandlerclass(convert_from) TrgClass = get_datahandlerclass(convert_to) if 'pairs' not in config: - config['pairs'] = SrcClass.ohclv_get_pairs(config['datadir'], config['ticker_interval']) + config['pairs'] = SrcClass.ohclv_get_pairs(Path(config['datadir']), config['ticker_interval']) logger.info(f"Converting OHLCV for {config['pairs']}") for pair in config['pairs']: - src = SrcClass(config['datadir'], pair) - trg = TrgClass(config['datadir'], pair) + src = SrcClass(Path(config['datadir']), pair) + trg = TrgClass(Path(config['datadir']), pair) data = src.ohlcv_load() logger.info(f"Converting {len(data)} candles for {pair}") trg.ohlcv_store(data) From ef0fcb0e0f5b82548b53380a0968d1750c64da69 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 10:21:30 +0100 Subject: [PATCH 007/317] Make data-finding safe --- freqtrade/data/datahandlers/jsondatahandler.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index 214958251..fe01176ad 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -19,8 +19,11 @@ class JsonDataHandler(IDataHandler): """ Returns a list of all pairs available in this datadir """ - return [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name)[0].replace('_', ' /') + + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name) for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] + # Check if regex found something and only return these results + return [match[0].replace('_', ' /') for match in _tmp if match] def ohlcv_store(self, timeframe: str, data: DataFrame): """ @@ -54,8 +57,10 @@ class JsonDataHandler(IDataHandler): """ Returns a list of all pairs available in this datadir """ - return [re.search(r'^(\S+)(?=\-trades.json)', p.name)[0].replace('_', '/') + _tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name) for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] + # Check if regex found something and only return these results to avoid exceptions. + return [match[0].replace('_', ' /') for match in _tmp if match] def trades_store(self, data: List[Dict]): """ From 3d4f62081e1cc9306a7eceb33fc575c9fd354c5d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 10:21:47 +0100 Subject: [PATCH 008/317] Allow timeframes for convert-data --- freqtrade/configuration/arguments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index a20ecaa43..888da4296 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -48,6 +48,7 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] ARGS_CONVERT_DATA = ["format_from", "format_to"] +ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", "timeframes", "erase"] @@ -261,7 +262,7 @@ class Arguments: parents=[_common_parser], ) convert_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=True)) - self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_data_cmd) + self._build_args(optionlist=ARGS_CONVERT_DATA_OHLCV, parser=convert_data_cmd) # Add convert-data subcommand convert_data_cmd = subparsers.add_parser( From 2a728ee68f669b0df51c70bf8e1f1a4cc52dcfc1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 10:25:30 +0100 Subject: [PATCH 009/317] fix bug in find-files --- freqtrade/data/datahandlers/jsondatahandler.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index fe01176ad..a6caf137d 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -23,7 +23,7 @@ class JsonDataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name) for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] # Check if regex found something and only return these results - return [match[0].replace('_', ' /') for match in _tmp if match] + return [match[0].replace('_', '/') for match in _tmp if match] def ohlcv_store(self, timeframe: str, data: DataFrame): """ @@ -42,8 +42,7 @@ class JsonDataHandler(IDataHandler): Load data for one pair :return: Dataframe """ - filename = JsonDataHandler._pair_data_filename(self.datadir, self._pair, - self._pair, timeframe) + filename = JsonDataHandler._pair_data_filename(self._datadir, self._pair, timeframe) pairdata = misc.file_load_json(filename) if not pairdata: return [] @@ -60,7 +59,7 @@ class JsonDataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name) for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] # Check if regex found something and only return these results to avoid exceptions. - return [match[0].replace('_', ' /') for match in _tmp if match] + return [match[0].replace('_', '/') for match in _tmp if match] def trades_store(self, data: List[Dict]): """ From 018e2703368459d06d0aab0b8e320321a12c6e2c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 10:28:11 +0100 Subject: [PATCH 010/317] Allow --pairs for convert arguments --- freqtrade/configuration/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 888da4296..cefa86927 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -47,7 +47,7 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] -ARGS_CONVERT_DATA = ["format_from", "format_to"] +ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", From d804372d74ad0a38a66ca6588abc2b01af04b341 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 10:28:25 +0100 Subject: [PATCH 011/317] Enhance ohlcv_convert method --- freqtrade/utils.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 4661e84b2..0805f0011 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -252,11 +252,12 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: if 'pairs' not in config: config['pairs'] = SrcClass.trades_get_pairs(Path(config['datadir'])) - logger.info(f"Converting trades for {config['pairs']}") + logger.info(f"Converting trades for {config['pairs']}") for pair in config['pairs']: - src = SrcClass(Path(config['datadir'], pair)) - trg = TrgClass(Path(config['datadir'], pair)) + print(pair) + src = SrcClass(Path(config['datadir']), pair) + trg = TrgClass(Path(config['datadir']), pair) data = src.trades_load() logger.info(f"Converting {len(data)} trades for {pair}") trg.trades_store(data) @@ -269,17 +270,25 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: """ SrcClass = get_datahandlerclass(convert_from) TrgClass = get_datahandlerclass(convert_to) + timeframes = config.get('timeframes', [config.get('ticker_interval')]) + logger.info(f"Converting OHLCV for timeframe {timeframes}") if 'pairs' not in config: - config['pairs'] = SrcClass.ohclv_get_pairs(Path(config['datadir']), config['ticker_interval']) - logger.info(f"Converting OHLCV for {config['pairs']}") + config['pairs'] = [] + # Check timeframes or fall back to ticker_interval. + for timeframe in timeframes: + config['pairs'].extend(SrcClass.ohlcv_get_pairs(Path(config['datadir']), + timeframe)) + logger.info(f"Converting OHLCV for {config['pairs']}") - for pair in config['pairs']: - src = SrcClass(Path(config['datadir']), pair) - trg = TrgClass(Path(config['datadir']), pair) - data = src.ohlcv_load() - logger.info(f"Converting {len(data)} candles for {pair}") - trg.ohlcv_store(data) + for timeframe in timeframes: + + for pair in config['pairs']: + src = SrcClass(Path(config['datadir']), pair) + trg = TrgClass(Path(config['datadir']), pair) + data = src.ohlcv_load(timeframe) + logger.info(f"Converting {len(data)} candles for {pair}") + # trg.ohlcv_store(data) def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: From abc6b9459a8133c4c66f3a2c9f7afc407af35a9c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 10:30:35 +0100 Subject: [PATCH 012/317] Add ohlcv_store call to convert_ohlcv --- freqtrade/data/datahandlers/jsondatahandler.py | 4 ++-- freqtrade/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index a6caf137d..747d80c54 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -25,13 +25,13 @@ class JsonDataHandler(IDataHandler): # Check if regex found something and only return these results return [match[0].replace('_', '/') for match in _tmp if match] - def ohlcv_store(self, timeframe: str, data: DataFrame): + def ohlcv_store(self, timeframe: str, data: DataFrame) -> None: """ Store data """ raise NotImplementedError() - def ohlcv_append(self, timeframe: str, data: DataFrame): + def ohlcv_append(self, timeframe: str, data: DataFrame) -> None: """ Append data to existing files """ diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 0805f0011..72b2afc8e 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -286,9 +286,9 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: for pair in config['pairs']: src = SrcClass(Path(config['datadir']), pair) trg = TrgClass(Path(config['datadir']), pair) - data = src.ohlcv_load(timeframe) + data = src.ohlcv_load(timeframe=timeframe) logger.info(f"Converting {len(data)} candles for {pair}") - # trg.ohlcv_store(data) + trg.ohlcv_store(timeframe=timeframe, data=data) def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: From 8f214aec891a862dab943de5f3561025852ac1fd Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 10:35:23 +0100 Subject: [PATCH 013/317] Fix "dumping" message to work correctly for .gz files --- freqtrade/misc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index bcba78cf0..ed37ace3a 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -47,14 +47,16 @@ def file_dump_json(filename: Path, data, is_zip=False) -> None: :param data: JSON Data to save :return: """ - logger.info(f'dumping json to "{filename}"') if is_zip: if filename.suffix != '.gz': filename = filename.with_suffix('.gz') + logger.info(f'dumping json to "{filename}"') + with gzip.open(filename, 'w') as fp: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) else: + logger.info(f'dumping json to "{filename}"') with open(filename, 'w') as fp: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) From c6d6dbfdb1646279fb439197e40716780bd99867 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 10:37:31 +0100 Subject: [PATCH 014/317] Implement jsondatahandler file store --- freqtrade/data/datahandlers/jsondatahandler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index 747d80c54..ec4e41ae8 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -29,7 +29,8 @@ class JsonDataHandler(IDataHandler): """ Store data """ - raise NotImplementedError() + filename = JsonDataHandler._pair_data_filename(self._datadir, self._pair, timeframe) + misc.file_dump_json(filename, data, is_zip=self._use_zip) def ohlcv_append(self, timeframe: str, data: DataFrame) -> None: """ From eff5cc0568b5a3464eb1326afccb3b0eac3a84fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 10:41:04 +0100 Subject: [PATCH 015/317] Add default to internals --- freqtrade/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 2f60bef2d..9c7e3db63 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -210,6 +210,7 @@ CONF_SCHEMA = { 'forcebuy_enable': {'type': 'boolean'}, 'internals': { 'type': 'object', + 'default': {}, 'properties': { 'process_throttle_secs': {'type': 'integer'}, 'interval': {'type': 'integer'}, From e529a4c261e0762056bb2c685170a3a9e3f5b653 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 11:08:11 +0100 Subject: [PATCH 016/317] Fix typehint for get_datahandlerclass --- freqtrade/data/datahandlers/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/datahandlers/__init__.py b/freqtrade/data/datahandlers/__init__.py index b3aa2e3a4..b22b61417 100644 --- a/freqtrade/data/datahandlers/__init__.py +++ b/freqtrade/data/datahandlers/__init__.py @@ -1,7 +1,9 @@ +from typing import Type + from .idatahandler import IDataHandler -def get_datahandlerclass(datatype: str) -> IDataHandler: +def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: """ Get datahandler class. Could be done using Resolvers, but since this may be called often and resolvers From 48728e2d666d3cddafc654b87a0b1e21cc1ea58c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 11:08:49 +0100 Subject: [PATCH 017/317] Change DataProvider interface to accept pair per method --- .../data/datahandlers/jsondatahandler.py | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index ec4e41ae8..ca6f94c5e 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -6,6 +6,7 @@ from pandas import DataFrame from freqtrade import misc from freqtrade.configuration import TimeRange +from freqtrade.data.converter import parse_ticker_dataframe from .idatahandler import IDataHandler @@ -25,31 +26,40 @@ class JsonDataHandler(IDataHandler): # Check if regex found something and only return these results return [match[0].replace('_', '/') for match in _tmp if match] - def ohlcv_store(self, timeframe: str, data: DataFrame) -> None: + def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: """ Store data """ - filename = JsonDataHandler._pair_data_filename(self._datadir, self._pair, timeframe) + filename = JsonDataHandler._pair_data_filename(self._datadir, pair, timeframe) misc.file_dump_json(filename, data, is_zip=self._use_zip) - def ohlcv_append(self, timeframe: str, data: DataFrame) -> None: + def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: """ Append data to existing files """ raise NotImplementedError() - def ohlcv_load(self, timeframe: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def _ohlcv_load(self, pair: str, timeframe: str, + timerange: Optional[TimeRange] = None, + fill_up_missing: bool = True, + drop_incomplete: bool = True, + ) -> DataFrame: """ - Load data for one pair + Load data for one pair from disk. + Implements the loading and conversation to a Pandas dataframe. :return: Dataframe """ - filename = JsonDataHandler._pair_data_filename(self._datadir, self._pair, timeframe) + filename = JsonDataHandler._pair_data_filename(self._datadir, pair, timeframe) pairdata = misc.file_load_json(filename) if not pairdata: - return [] + return DataFrame() if timerange: pairdata = IDataHandler.trim_tickerlist(pairdata, timerange) + return parse_ticker_dataframe(pairdata, timeframe, + pair=self._pair, + fill_missing=fill_up_missing, + drop_incomplete=drop_incomplete) return pairdata @classmethod @@ -62,26 +72,26 @@ class JsonDataHandler(IDataHandler): # Check if regex found something and only return these results to avoid exceptions. return [match[0].replace('_', '/') for match in _tmp if match] - def trades_store(self, data: List[Dict]): + def trades_store(self, pair: str, data: List[Dict]): """ Store data """ - filename = self._pair_trades_filename(self._datadir, self._pair) + filename = self._pair_trades_filename(self._datadir, pair) misc.file_dump_json(filename, data, is_zip=self._use_zip) - def trades_append(self, data: DataFrame): + def trades_append(self, pair: str, data: DataFrame): """ Append data to existing files """ raise NotImplementedError() - def trades_load(self, timerange: Optional[TimeRange] = None) -> List[Dict]: + def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]: """ Load a pair from file, either .json.gz or .json # TODO: validate timerange ... :return: List of trades """ - filename = self._pair_trades_filename(self._datadir, self._pair) + filename = self._pair_trades_filename(self._datadir, pair) tradesdata = misc.file_load_json(filename) if not tradesdata: return [] From d923bab8288f65a4c97bf3340883f1cb11f60523 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 11:09:29 +0100 Subject: [PATCH 018/317] Remove abstract interface for now --- freqtrade/data/datahandlers/idatahandler.py | 106 +++++++++++--------- 1 file changed, 56 insertions(+), 50 deletions(-) diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/datahandlers/idatahandler.py index ffe50b14e..b82898353 100644 --- a/freqtrade/data/datahandlers/idatahandler.py +++ b/freqtrade/data/datahandlers/idatahandler.py @@ -3,71 +3,77 @@ Abstract datahandler interface. It's subclasses handle and storing data from disk. """ - +import logging from abc import ABC, abstractmethod, abstractclassmethod from pathlib import Path from typing import Dict, List, Optional - +from copy import deepcopy from pandas import DataFrame from freqtrade.configuration import TimeRange +from freqtrade.exchange import timeframe_to_seconds +from freqtrade.data.converter import parse_ticker_dataframe + +logger = logging.getLogger(__name__) class IDataHandler(ABC): - def __init__(self, datadir: Path, pair: str) -> None: + def __init__(self, datadir: Path) -> None: self._datadir = datadir - self._pair = pair - @abstractclassmethod - def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + # TODO: create abstract interface + + def ohlcv_load(self, pair, timeframe: str, + timerange: Optional[TimeRange] = None, + fill_up_missing: bool = True, + drop_incomplete: bool = True, + startup_candles: int = 0, + ) -> DataFrame: """ - Returns a list of all pairs available in this datadir + Load cached ticker history for the given pair. + + :param pair: Pair to load data for + :param timeframe: Ticker timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange + :param fill_up_missing: Fill missing values with "No action"-candles + :param drop_incomplete: Drop last candle assuming it may be incomplete. + :param startup_candles: Additional candles to load at the start of the period + :return: DataFrame with ohlcv data, or empty DataFrame + """ + # Fix startup period + timerange_startup = deepcopy(timerange) + if startup_candles > 0 and timerange_startup: + timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) + + pairdf = self._ohlcv_load(pair, timeframe, + timerange=timerange_startup, + fill_missing=fill_up_missing, + drop_incomplete=drop_incomplete) + if pairdf.empty(): + logger.warning( + f'No history data for pair: "{pair}", timeframe: {timeframe}. ' + 'Use `freqtrade download-data` to download the data' + ) + return pairdf + else: + if timerange_startup: + self._validate_pairdata(pair, pairdf, timerange_startup) + return pairdf + + def _validate_pairdata(pair, pairdata: DataFrame, timerange: TimeRange): + """ + Validates pairdata for missing data at start end end and logs warnings. + :param pairdata: Dataframe to validate + :param timerange: Timerange specified for start and end dates """ - @abstractmethod - def ohlcv_store(self, timeframe: str, data: DataFrame): - """ - Store data - """ - - @abstractmethod - def ohlcv_append(self, timeframe: str, data: DataFrame): - """ - Append data to existing files - """ - - @abstractmethod - def ohlcv_load(self, timeframe: str, timerange: Optional[TimeRange] = None) -> DataFrame: - """ - Load data for one pair - :return: Dataframe - """ - - @abstractclassmethod - def trades_get_pairs(cls, datadir: Path) -> List[str]: - """ - Returns a list of all pairs available in this datadir - """ - - @abstractmethod - def trades_store(self, data: DataFrame): - """ - Store data - """ - - @abstractmethod - def trades_append(self, data: DataFrame): - """ - Append data to existing files - """ - - @abstractmethod - def trades_load(self, timerange: Optional[TimeRange] = None): - """ - Load data for one pair - :return: Dataframe - """ + if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: + logger.warning('Missing data at start for pair %s, data starts at %s', + pair, arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) + if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: + logger.warning('Missing data at end for pair %s, data ends at %s', + pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) @staticmethod def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: From 1b90ec58b9bf621b9a6b2941597d5d8efbb118ee Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 11:09:59 +0100 Subject: [PATCH 019/317] Use changed pair-handling for providers --- freqtrade/data/datahandlers/idatahandler.py | 15 ++++++++------- freqtrade/utils.py | 20 +++++++++----------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/datahandlers/idatahandler.py index b82898353..a1413fe98 100644 --- a/freqtrade/data/datahandlers/idatahandler.py +++ b/freqtrade/data/datahandlers/idatahandler.py @@ -4,15 +4,16 @@ It's subclasses handle and storing data from disk. """ import logging -from abc import ABC, abstractmethod, abstractclassmethod +from abc import ABC, abstractclassmethod, abstractmethod +from copy import deepcopy from pathlib import Path from typing import Dict, List, Optional -from copy import deepcopy + +import arrow from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.exchange import timeframe_to_seconds -from freqtrade.data.converter import parse_ticker_dataframe logger = logging.getLogger(__name__) @@ -26,7 +27,7 @@ class IDataHandler(ABC): def ohlcv_load(self, pair, timeframe: str, timerange: Optional[TimeRange] = None, - fill_up_missing: bool = True, + fill_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, ) -> DataFrame: @@ -48,9 +49,9 @@ class IDataHandler(ABC): pairdf = self._ohlcv_load(pair, timeframe, timerange=timerange_startup, - fill_missing=fill_up_missing, + fill_missing=fill_missing, drop_incomplete=drop_incomplete) - if pairdf.empty(): + if pairdf.empty: logger.warning( f'No history data for pair: "{pair}", timeframe: {timeframe}. ' 'Use `freqtrade download-data` to download the data' @@ -61,7 +62,7 @@ class IDataHandler(ABC): self._validate_pairdata(pair, pairdf, timerange_startup) return pairdf - def _validate_pairdata(pair, pairdata: DataFrame, timerange: TimeRange): + def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange): """ Validates pairdata for missing data at start end end and logs warnings. :param pairdata: Dataframe to validate diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 72b2afc8e..25f10f71a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -253,14 +253,12 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: if 'pairs' not in config: config['pairs'] = SrcClass.trades_get_pairs(Path(config['datadir'])) logger.info(f"Converting trades for {config['pairs']}") - + src = SrcClass(Path(config['datadir'])) + trg = TrgClass(Path(config['datadir'])) for pair in config['pairs']: - print(pair) - src = SrcClass(Path(config['datadir']), pair) - trg = TrgClass(Path(config['datadir']), pair) - data = src.trades_load() + data = src.trades_load(pair=pair) logger.info(f"Converting {len(data)} trades for {pair}") - trg.trades_store(data) + trg.trades_store(pair, data) def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str): @@ -281,14 +279,14 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: timeframe)) logger.info(f"Converting OHLCV for {config['pairs']}") - for timeframe in timeframes: + src = SrcClass(Path(config['datadir'])) + trg = TrgClass(Path(config['datadir'])) + for timeframe in timeframes: for pair in config['pairs']: - src = SrcClass(Path(config['datadir']), pair) - trg = TrgClass(Path(config['datadir']), pair) - data = src.ohlcv_load(timeframe=timeframe) + data = src.ohlcv_load(pair=pair, timeframe=timeframe) logger.info(f"Converting {len(data)} candles for {pair}") - trg.ohlcv_store(timeframe=timeframe, data=data) + trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: From d9e7d64f33149d26f8517ba594f044a9cc07ca5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 13:24:09 +0100 Subject: [PATCH 020/317] Split parse_ticker_dataframe some logic to clean_ohlcv_dataframe. --- freqtrade/data/converter.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index e45dd451e..6b44a6b20 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -37,9 +37,29 @@ def parse_ticker_dataframe(ticker: list, timeframe: str, pair: str, *, # and fail with exception... frame = frame.astype(dtype={'open': 'float', 'high': 'float', 'low': 'float', 'close': 'float', 'volume': 'float'}) + return clean_ohlcv_dataframe(frame, timeframe, pair, + fill_missing=fill_missing, + drop_incomplete=drop_incomplete) + +def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *, + fill_missing: bool = True, + drop_incomplete: bool = True) -> DataFrame: + """ + Clense a ohlcv dataframe by + * Grouping it by date (removes duplicate tics) + * dropping last candles if requested + * Filling up missing data (if requested) + :param data: DataFrame containing ohlcv data. + :param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data + :param pair: Pair this data is for (used to warn if fillup was necessary) + :param fill_missing: fill up missing candles with 0 candles + (see ohlcv_fill_up_missing_data for details) + :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete + :return: DataFrame + """ # group by index and aggregate results to eliminate duplicate ticks - frame = frame.groupby(by='date', as_index=False, sort=True).agg({ + data = data.groupby(by='date', as_index=False, sort=True).agg({ 'open': 'first', 'high': 'max', 'low': 'min', @@ -48,13 +68,13 @@ def parse_ticker_dataframe(ticker: list, timeframe: str, pair: str, *, }) # eliminate partial candle if drop_incomplete: - frame.drop(frame.tail(1).index, inplace=True) + data.drop(data.tail(1).index, inplace=True) logger.debug('Dropping last candle') if fill_missing: - return ohlcv_fill_up_missing_data(frame, timeframe, pair) + return ohlcv_fill_up_missing_data(data, timeframe, pair) else: - return frame + return data def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) -> DataFrame: From 377d59abe70fb93b1d644a39412d859853b97cf0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 15:04:45 +0100 Subject: [PATCH 021/317] Be selective how to load ohclv data for conversation --- freqtrade/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 25f10f71a..877e4825c 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -284,7 +284,10 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: for timeframe in timeframes: for pair in config['pairs']: - data = src.ohlcv_load(pair=pair, timeframe=timeframe) + data = src.ohlcv_load(pair=pair, timeframe=timeframe, + fill_missing=False, + drop_incomplete=False, + startup_candles=0) logger.info(f"Converting {len(data)} candles for {pair}") trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) @@ -294,8 +297,6 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: Convert data from one format to another """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - from pprint import pprint - pprint(config) if ohlcv: convert_ohlcv_format(config, convert_from=args['format_from'], convert_to=args['format_to']) From 866908d2ca3a2a5caa2792c23f720b4097acced8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 15:05:01 +0100 Subject: [PATCH 022/317] Load and save using pandas internal function --- .../data/datahandlers/jsondatahandler.py | 62 +++++++++++++------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index ca6f94c5e..3e64885a7 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -2,11 +2,12 @@ import re from pathlib import Path from typing import Dict, List, Optional -from pandas import DataFrame +import numpy as np +from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange -from freqtrade.data.converter import parse_ticker_dataframe +from freqtrade.data.converter import clean_ohlcv_dataframe from .idatahandler import IDataHandler @@ -14,6 +15,7 @@ from .idatahandler import IDataHandler class JsonDataHandler(IDataHandler): _use_zip = False + _columns = ['date', 'open', 'high', 'low', 'close', 'volume'] @classmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: @@ -28,20 +30,27 @@ class JsonDataHandler(IDataHandler): def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: """ - Store data + Store data in json format "values". + format looks as follows: + [[,,,,]] + :param pair: Pair - used to generate filename + :timeframe: Timeframe - used to generate filename + :data: Dataframe containing OHLCV data + :return: None """ - filename = JsonDataHandler._pair_data_filename(self._datadir, pair, timeframe) - misc.file_dump_json(filename, data, is_zip=self._use_zip) + filename = self._pair_data_filename(self._datadir, pair, timeframe) + _data = data.copy() + # Convert date to int + _data['date'] = _data['date'].astype(np.int64) // 1000 // 1000 - def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: - """ - Append data to existing files - """ - raise NotImplementedError() + # Reset index, select only appropriate columns and save as json + _data.reset_index(drop=True).loc[:, self._columns].to_json( + filename, orient="values", + compression='gzip' if self._use_zip else None) def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, - fill_up_missing: bool = True, + fill_missing: bool = True, drop_incomplete: bool = True, ) -> DataFrame: """ @@ -49,18 +58,31 @@ class JsonDataHandler(IDataHandler): Implements the loading and conversation to a Pandas dataframe. :return: Dataframe """ - filename = JsonDataHandler._pair_data_filename(self._datadir, pair, timeframe) - pairdata = misc.file_load_json(filename) - if not pairdata: - return DataFrame() + filename = self._pair_data_filename(self._datadir, pair, timeframe) + pairdata = read_json(filename, orient='values') + pairdata.columns = self._columns + pairdata['date'] = to_datetime(pairdata['date'], + unit='ms', + utc=True, + infer_datetime_format=True) if timerange: pairdata = IDataHandler.trim_tickerlist(pairdata, timerange) - return parse_ticker_dataframe(pairdata, timeframe, - pair=self._pair, - fill_missing=fill_up_missing, - drop_incomplete=drop_incomplete) - return pairdata + + return clean_ohlcv_dataframe(pairdata, timeframe, + pair=pair, + fill_missing=fill_missing, + drop_incomplete=drop_incomplete) + + def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Append data to existing data structures + :param pair: Pair + :param timeframe: Timeframe this ohlcv data is for + :param data: Data to append. + + """ + raise NotImplementedError() @classmethod def trades_get_pairs(cls, datadir: Path) -> List[str]: From db520a09ee1f3ef44d29e98dad7225be17112ec5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 15:07:49 +0100 Subject: [PATCH 023/317] Trim dataframe, not tickerlist --- freqtrade/data/datahandlers/idatahandler.py | 29 +------------------ .../data/datahandlers/jsondatahandler.py | 13 +++++++-- freqtrade/utils.py | 1 + 3 files changed, 12 insertions(+), 31 deletions(-) diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/datahandlers/idatahandler.py index a1413fe98..b9d11f243 100644 --- a/freqtrade/data/datahandlers/idatahandler.py +++ b/freqtrade/data/datahandlers/idatahandler.py @@ -37,7 +37,7 @@ class IDataHandler(ABC): :param pair: Pair to load data for :param timeframe: Ticker timeframe (e.g. "5m") :param timerange: Limit data to be loaded to this timerange - :param fill_up_missing: Fill missing values with "No action"-candles + :param fill_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :return: DataFrame with ohlcv data, or empty DataFrame @@ -75,30 +75,3 @@ class IDataHandler(ABC): if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: logger.warning('Missing data at end for pair %s, data ends at %s', pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - - @staticmethod - def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: - """ - TODO: investigate if this is needed ... we can probably cover this in a dataframe - Trim tickerlist based on given timerange - """ - if not tickerlist: - return tickerlist - - start_index = 0 - stop_index = len(tickerlist) - - if timerange.starttype == 'date': - while (start_index < len(tickerlist) and - tickerlist[start_index][0] < timerange.startts * 1000): - start_index += 1 - - if timerange.stoptype == 'date': - while (stop_index > 0 and - tickerlist[stop_index-1][0] > timerange.stopts * 1000): - stop_index -= 1 - - if start_index > stop_index: - raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') - - return tickerlist[start_index:stop_index] diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index 3e64885a7..215aa003a 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -8,6 +8,7 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.data.converter import clean_ohlcv_dataframe +from freqtrade.data.history import trim_dataframe from .idatahandler import IDataHandler @@ -54,9 +55,15 @@ class JsonDataHandler(IDataHandler): drop_incomplete: bool = True, ) -> DataFrame: """ - Load data for one pair from disk. + Internal method used to load data for one pair from disk. Implements the loading and conversation to a Pandas dataframe. - :return: Dataframe + :param pair: Pair to load data for + :param timeframe: Ticker timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange + :param fill_missing: Fill missing values with "No action"-candles + :param drop_incomplete: Drop last candle assuming it may be incomplete. + :param startup_candles: Additional candles to load at the start of the period + :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename(self._datadir, pair, timeframe) pairdata = read_json(filename, orient='values') @@ -67,7 +74,7 @@ class JsonDataHandler(IDataHandler): infer_datetime_format=True) if timerange: - pairdata = IDataHandler.trim_tickerlist(pairdata, timerange) + pairdata = trim_dataframe(pairdata, timerange) return clean_ohlcv_dataframe(pairdata, timeframe, pair=pair, diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 877e4825c..5f646cc7a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -285,6 +285,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: for timeframe in timeframes: for pair in config['pairs']: data = src.ohlcv_load(pair=pair, timeframe=timeframe, + timerange=None, fill_missing=False, drop_incomplete=False, startup_candles=0) From 873f5dbe6bbe73e6c155dd451d6fa553894276fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 15:13:17 +0100 Subject: [PATCH 024/317] Revrite validate_pairdata to work with pandas --- freqtrade/data/datahandlers/idatahandler.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/datahandlers/idatahandler.py index b9d11f243..9508e022e 100644 --- a/freqtrade/data/datahandlers/idatahandler.py +++ b/freqtrade/data/datahandlers/idatahandler.py @@ -8,8 +8,7 @@ from abc import ABC, abstractclassmethod, abstractmethod from copy import deepcopy from pathlib import Path from typing import Dict, List, Optional - -import arrow +from datetime import datetime, timezone from pandas import DataFrame from freqtrade.configuration import TimeRange @@ -70,8 +69,12 @@ class IDataHandler(ABC): """ if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: - logger.warning('Missing data at start for pair %s, data starts at %s', - pair, arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + if pairdata.iloc[0]['date'] > start: + logger.warning(f"Missing data at start for pair {pair}, " + f"data starts at {pairdata.iloc[0]['date']}") if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: - logger.warning('Missing data at end for pair %s, data ends at %s', - pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) + stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) + if pairdata.iloc[-1]['date'] < stop: + logger.warning(f"Missing data at end for pair {pair}," + f"data ends at {pairdata.iloc[-1]['date']}") From 53ee636fa02880d12ef3c32430e62d1af3f7a92f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 15:24:53 +0100 Subject: [PATCH 025/317] Check if file exists before loading --- freqtrade/data/datahandlers/jsondatahandler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index 215aa003a..2893393b1 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -66,6 +66,8 @@ class JsonDataHandler(IDataHandler): :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename(self._datadir, pair, timeframe) + if not filename.is_file(): + return DataFrame(columns=self._columns) pairdata = read_json(filename, orient='values') pairdata.columns = self._columns pairdata['date'] = to_datetime(pairdata['date'], From 88fa7fc24cd0688b0d0d58d6fa572e2f0f45b869 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 15:35:59 +0100 Subject: [PATCH 026/317] Simplify validate dataframe method --- freqtrade/data/datahandlers/idatahandler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/datahandlers/idatahandler.py index 9508e022e..c28d03930 100644 --- a/freqtrade/data/datahandlers/idatahandler.py +++ b/freqtrade/data/datahandlers/idatahandler.py @@ -68,13 +68,13 @@ class IDataHandler(ABC): :param timerange: Timerange specified for start and end dates """ - if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: + if timerange.starttype == 'date': start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) if pairdata.iloc[0]['date'] > start: logger.warning(f"Missing data at start for pair {pair}, " - f"data starts at {pairdata.iloc[0]['date']}") - if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: + f"data starts at {pairdata.iloc[0]['date']:%Y-%m-%d %H:%M:%S}") + if timerange.stoptype == 'date': stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) if pairdata.iloc[-1]['date'] < stop: - logger.warning(f"Missing data at end for pair {pair}," - f"data ends at {pairdata.iloc[-1]['date']}") + logger.warning(f"Missing data at end for pair {pair}, " + f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}") From 9d8ea2f13be579e44a03b93c35b307a09f7fc98e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 15:40:42 +0100 Subject: [PATCH 027/317] Replace calls to load_tickerdata_file with DataHandler calls --- freqtrade/data/history.py | 37 +++++++++++++++---------------------- tests/data/test_history.py | 9 +++++---- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 4c5c0521f..f09fe3d6a 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -8,7 +8,6 @@ Includes: import logging import operator -from copy import deepcopy from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -18,8 +17,9 @@ from pandas import DataFrame from freqtrade import OperationalException, misc from freqtrade.configuration import TimeRange -from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv -from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds +from freqtrade.data.converter import trades_to_ohlcv +from freqtrade.exchange import Exchange, timeframe_to_minutes +from .datahandlers import get_datahandlerclass logger = logging.getLogger(__name__) @@ -126,11 +126,12 @@ def _validate_pairdata(pair, pairdata, timerange: TimeRange): def load_pair_history(pair: str, timeframe: str, - datadir: Path, + datadir: Path, *, timerange: Optional[TimeRange] = None, fill_up_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, + data_format: str = 'json' ) -> DataFrame: """ Load cached ticker history for the given pair. @@ -142,26 +143,18 @@ def load_pair_history(pair: str, :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period + :param data_format: Format of the data :return: DataFrame with ohlcv data, or empty DataFrame """ - timerange_startup = deepcopy(timerange) - if startup_candles > 0 and timerange_startup: - timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) - - pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) - - if pairdata: - if timerange_startup: - _validate_pairdata(pair, pairdata, timerange_startup) - return parse_ticker_dataframe(pairdata, timeframe, pair=pair, - fill_missing=fill_up_missing, - drop_incomplete=drop_incomplete) - else: - logger.warning( - f'No history data for pair: "{pair}", timeframe: {timeframe}. ' - 'Use `freqtrade download-data` to download the data' - ) - return DataFrame() + HandlerClass = get_datahandlerclass(data_format) + loader = HandlerClass(datadir) + return loader.ohlcv_load(pair=pair, + timeframe=timeframe, + timerange=timerange, + fill_missing=fill_up_missing, + drop_incomplete=drop_incomplete, + startup_candles=startup_candles, + ) def load_data(datadir: Path, diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 7b3143db9..f13e386f8 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -96,8 +96,9 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: - ltfmock = mocker.patch('freqtrade.data.history.load_tickerdata_file', - MagicMock(return_value=None)) + ltfmock = mocker.patch( + 'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler._ohlcv_load', + MagicMock(return_value=DataFrame())) timerange = TimeRange('date', None, 1510639620, 0) load_pair_history(pair='UNITTEST/BTC', timeframe='1m', datadir=testdatadir, timerange=timerange, @@ -361,8 +362,8 @@ def test_load_partial_missing(testdatadir, caplog) -> None: # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(tickerdata['UNITTEST/BTC']) - # Shift endtime with +5 - as last candle is dropped (partial candle) - end_real = arrow.get(tickerdata['UNITTEST/BTC'].iloc[-1, 0]).shift(minutes=5) + # This validation happens now after parsing to pandas. + end_real = arrow.get(tickerdata['UNITTEST/BTC'].iloc[-1, 0]) assert log_has(f'Missing data at end for pair ' f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}', caplog) From 416517b0c901f3438dcdc1af54fe5b32a77e38ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 15:47:04 +0100 Subject: [PATCH 028/317] Move trim_dataframe from history to converter --- freqtrade/data/converter.py | 19 ++++++++ .../data/datahandlers/jsondatahandler.py | 3 +- freqtrade/data/history.py | 26 ---------- freqtrade/optimize/backtesting.py | 3 +- freqtrade/optimize/hyperopt.py | 3 +- freqtrade/plot/plotting.py | 3 +- tests/data/test_converter.py | 47 ++++++++++++++++++- tests/data/test_history.py | 46 +----------------- 8 files changed, 73 insertions(+), 77 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 6b44a6b20..52ce3c593 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -2,10 +2,12 @@ Functions to convert data from one format to another """ import logging +from datetime import datetime, timezone import pandas as pd from pandas import DataFrame, to_datetime +from freqtrade.configuration.timerange import TimeRange logger = logging.getLogger(__name__) @@ -112,6 +114,23 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) return df +def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date') -> DataFrame: + """ + Trim dataframe based on given timerange + :param df: Dataframe to trim + :param timerange: timerange (use start and end date if available) + :param: df_date_col: Column in the dataframe to use as Date column + :return: trimmed dataframe + """ + if timerange.starttype == 'date': + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + df = df.loc[df[df_date_col] >= start, :] + if timerange.stoptype == 'date': + stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) + df = df.loc[df[df_date_col] <= stop, :] + return df + + def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: """ Gets order book list, returns dataframe with below format per suggested by creslin diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index 2893393b1..c799784d0 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -7,8 +7,7 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange -from freqtrade.data.converter import clean_ohlcv_dataframe -from freqtrade.data.history import trim_dataframe +from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe from .idatahandler import IDataHandler diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index f09fe3d6a..cc82217a0 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -50,23 +50,6 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: return tickerlist[start_index:stop_index] -def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date') -> DataFrame: - """ - Trim dataframe based on given timerange - :param df: Dataframe to trim - :param timerange: timerange (use start and end date if available) - :param: df_date_col: Column in the dataframe to use as Date column - :return: trimmed dataframe - """ - if timerange.starttype == 'date': - start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) - df = df.loc[df[df_date_col] >= start, :] - if timerange.stoptype == 'date': - stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) - df = df.loc[df[df_date_col] <= stop, :] - return df - - def load_tickerdata_file(datadir: Path, pair: str, timeframe: str, timerange: Optional[TimeRange] = None) -> List[Dict]: """ @@ -115,15 +98,6 @@ def store_trades_file(datadir: Path, pair: str, misc.file_dump_json(filename, data, is_zip=is_zip) -def _validate_pairdata(pair, pairdata, timerange: TimeRange): - if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: - logger.warning('Missing data at start for pair %s, data starts at %s', - pair, arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: - logger.warning('Missing data at end for pair %s, data ends at %s', - pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - - def load_pair_history(pair: str, timeframe: str, datadir: Path, *, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a8fe90a06..98ee71a60 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -16,6 +16,7 @@ from freqtrade import OperationalException from freqtrade.configuration import (TimeRange, remove_credentials, validate_config_consistency) from freqtrade.data import history +from freqtrade.data.converter import trim_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import file_dump_json @@ -482,7 +483,7 @@ class Backtesting: # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): - preprocessed[pair] = history.trim_dataframe(df, timerange) + preprocessed[pair] = trim_dataframe(df, timerange) min_date, max_date = history.get_timerange(preprocessed) logger.info( diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 48f883ac5..d29508b49 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -23,7 +23,8 @@ from joblib import (Parallel, cpu_count, delayed, dump, load, from pandas import DataFrame from freqtrade import OperationalException -from freqtrade.data.history import get_timerange, trim_dataframe +from freqtrade.data.history import get_timerange +from freqtrade.data.converter import trim_dataframe from freqtrade.misc import plural, round_dict from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index db4637ee5..0ef71ed82 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List import pandas as pd from freqtrade.configuration import TimeRange from freqtrade.data import history +from freqtrade.data.converter import trim_dataframe from freqtrade.data.btanalysis import (combine_tickers_with_mean, create_cum_profit, extract_trades_of_period, load_trades) @@ -47,7 +48,7 @@ def init_plotscript(config): db_url=config.get('db_url'), exportfilename=config.get('exportfilename'), ) - trades = history.trim_dataframe(trades, timerange, 'open_time') + trades = trim_dataframe(trades, timerange, 'open_time') return {"tickers": tickers, "trades": trades, "pairs": pairs, diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 414551c95..eb8a8e513 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -1,8 +1,11 @@ # pragma pylint: disable=missing-docstring, C0103 import logging -from freqtrade.data.converter import parse_ticker_dataframe, ohlcv_fill_up_missing_data -from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timerange +from freqtrade.configuration.timerange import TimeRange +from freqtrade.data.converter import (ohlcv_fill_up_missing_data, + parse_ticker_dataframe, trim_dataframe) +from freqtrade.data.history import (get_timerange, load_data, + load_pair_history, validate_backtest_data) from tests.conftest import log_has @@ -145,3 +148,43 @@ def test_ohlcv_drop_incomplete(caplog): assert len(data) == 3 assert log_has("Dropping last candle", caplog) + + +def test_trim_dataframe(testdatadir) -> None: + data = load_data( + datadir=testdatadir, + timeframe='1m', + pairs=['UNITTEST/BTC'] + )['UNITTEST/BTC'] + min_date = int(data.iloc[0]['date'].timestamp()) + max_date = int(data.iloc[-1]['date'].timestamp()) + data_modify = data.copy() + + # Remove first 30 minutes (1800 s) + tr = TimeRange('date', None, min_date + 1800, 0) + data_modify = trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 30 + assert all(data_modify.iloc[-1] == data.iloc[-1]) + assert all(data_modify.iloc[0] == data.iloc[30]) + + data_modify = data.copy() + # Remove last 30 minutes (1800 s) + tr = TimeRange(None, 'date', 0, max_date - 1800) + data_modify = trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 30 + assert all(data_modify.iloc[0] == data.iloc[0]) + assert all(data_modify.iloc[-1] == data.iloc[-31]) + + data_modify = data.copy() + # Remove first 25 and last 30 minutes (1800 s) + tr = TimeRange('date', 'date', min_date + 1500, max_date - 1800) + data_modify = trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 55 + # first row matches 25th original row + assert all(data_modify.iloc[0] == data.iloc[25]) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index f13e386f8..5f7d0f5a2 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -18,10 +18,8 @@ from freqtrade.data.history import (_download_pair_history, load_tickerdata_file, pair_data_filename, pair_trades_filename, refresh_backtest_ohlcv_data, - refresh_backtest_trades_data, - refresh_data, - trim_dataframe, trim_tickerlist, - validate_backtest_data) + refresh_backtest_trades_data, refresh_data, + trim_tickerlist, validate_backtest_data) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.strategy.default_strategy import DefaultStrategy @@ -444,46 +442,6 @@ def test_trim_tickerlist(testdatadir) -> None: assert not ticker -def test_trim_dataframe(testdatadir) -> None: - data = load_data( - datadir=testdatadir, - timeframe='1m', - pairs=['UNITTEST/BTC'] - )['UNITTEST/BTC'] - min_date = int(data.iloc[0]['date'].timestamp()) - max_date = int(data.iloc[-1]['date'].timestamp()) - data_modify = data.copy() - - # Remove first 30 minutes (1800 s) - tr = TimeRange('date', None, min_date + 1800, 0) - data_modify = trim_dataframe(data_modify, tr) - assert not data_modify.equals(data) - assert len(data_modify) < len(data) - assert len(data_modify) == len(data) - 30 - assert all(data_modify.iloc[-1] == data.iloc[-1]) - assert all(data_modify.iloc[0] == data.iloc[30]) - - data_modify = data.copy() - # Remove last 30 minutes (1800 s) - tr = TimeRange(None, 'date', 0, max_date - 1800) - data_modify = trim_dataframe(data_modify, tr) - assert not data_modify.equals(data) - assert len(data_modify) < len(data) - assert len(data_modify) == len(data) - 30 - assert all(data_modify.iloc[0] == data.iloc[0]) - assert all(data_modify.iloc[-1] == data.iloc[-31]) - - data_modify = data.copy() - # Remove first 25 and last 30 minutes (1800 s) - tr = TimeRange('date', 'date', min_date + 1500, max_date - 1800) - data_modify = trim_dataframe(data_modify, tr) - assert not data_modify.equals(data) - assert len(data_modify) < len(data) - assert len(data_modify) == len(data) - 55 - # first row matches 25th original row - assert all(data_modify.iloc[0] == data.iloc[25]) - - def test_file_dump_json_tofile(testdatadir) -> None: file = testdatadir / 'test_{id}.json'.format(id=str(uuid.uuid4())) data = {'bar': 'foo'} From 5fca17d7e1fc89f30e90156d2fc88f954d9faf82 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 15:55:28 +0100 Subject: [PATCH 029/317] Allow initializing handler-class just once --- freqtrade/data/history.py | 40 +++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index cc82217a0..7c5cf482d 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -8,7 +8,7 @@ Includes: import logging import operator -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -18,6 +18,7 @@ from pandas import DataFrame from freqtrade import OperationalException, misc from freqtrade.configuration import TimeRange from freqtrade.data.converter import trades_to_ohlcv +from freqtrade.data.datahandlers.idatahandler import IDataHandler from freqtrade.exchange import Exchange, timeframe_to_minutes from .datahandlers import get_datahandlerclass @@ -105,7 +106,8 @@ def load_pair_history(pair: str, fill_up_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, - data_format: str = 'json' + data_format: str = 'json', + data_handler: IDataHandler, ) -> DataFrame: """ Load cached ticker history for the given pair. @@ -117,18 +119,21 @@ def load_pair_history(pair: str, :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period - :param data_format: Format of the data + :param data_format: Format of the data. Ignored if data_handler is set. + :param data_handler: Initialized data-handler to use. + Will be initialized from data_format if not set :return: DataFrame with ohlcv data, or empty DataFrame """ - HandlerClass = get_datahandlerclass(data_format) - loader = HandlerClass(datadir) - return loader.ohlcv_load(pair=pair, - timeframe=timeframe, - timerange=timerange, - fill_missing=fill_up_missing, - drop_incomplete=drop_incomplete, - startup_candles=startup_candles, - ) + if not data_handler: + HandlerClass = get_datahandlerclass(data_format) + data_handler = HandlerClass(datadir) + return data_handler.ohlcv_load(pair=pair, + timeframe=timeframe, + timerange=timerange, + fill_missing=fill_up_missing, + drop_incomplete=drop_incomplete, + startup_candles=startup_candles, + ) def load_data(datadir: Path, @@ -137,7 +142,8 @@ def load_data(datadir: Path, timerange: Optional[TimeRange] = None, fill_up_missing: bool = True, startup_candles: int = 0, - fail_without_data: bool = False + fail_without_data: bool = False, + data_format: str = 'json', ) -> Dict[str, DataFrame]: """ Load ticker history data for a list of pairs. @@ -149,17 +155,23 @@ def load_data(datadir: Path, :param fill_up_missing: Fill missing values with "No action"-candles :param startup_candles: Additional candles to load at the start of the period :param fail_without_data: Raise OperationalException if no data is found. + :param data_handler: Initialized data-handler to use. :return: dict(:) """ result: Dict[str, DataFrame] = {} if startup_candles > 0 and timerange: logger.info(f'Using indicator startup period: {startup_candles} ...') + HandlerClass = get_datahandlerclass(data_format) + data_handler = HandlerClass(datadir) + for pair in pairs: hist = load_pair_history(pair=pair, timeframe=timeframe, datadir=datadir, timerange=timerange, fill_up_missing=fill_up_missing, - startup_candles=startup_candles) + startup_candles=startup_candles, + data_handler=data_handler + ) if not hist.empty: result[pair] = hist From 9547d47ae2f58bc416454bb0b5212c79da8845e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 16:12:20 +0100 Subject: [PATCH 030/317] Initialize datahandlers --- freqtrade/data/datahandlers/__init__.py | 16 +++++++++++- freqtrade/data/history.py | 34 ++++++++++++++----------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/freqtrade/data/datahandlers/__init__.py b/freqtrade/data/datahandlers/__init__.py index b22b61417..a21dd832d 100644 --- a/freqtrade/data/datahandlers/__init__.py +++ b/freqtrade/data/datahandlers/__init__.py @@ -1,5 +1,5 @@ from typing import Type - +from pathlib import Path from .idatahandler import IDataHandler @@ -20,3 +20,17 @@ def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: return JsonGzDataHandler else: raise ValueError(f"No datahandler for datatype {datatype} available.") + + +def get_datahandler(datadir: Path, data_format: str = None, + data_handler: IDataHandler = None) -> IDataHandler: + """ + :param datadir: Folder to save data + :data_format: dataformat to use + :data_handler: returns this datahandler if it exists or initializes a new one + """ + + if not data_handler: + HandlerClass = get_datahandlerclass(data_format or 'json') + data_handler = HandlerClass(datadir) + return data_handler diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 7c5cf482d..4f31ebd60 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -18,9 +18,9 @@ from pandas import DataFrame from freqtrade import OperationalException, misc from freqtrade.configuration import TimeRange from freqtrade.data.converter import trades_to_ohlcv +from freqtrade.data.datahandlers import get_datahandler, get_datahandlerclass from freqtrade.data.datahandlers.idatahandler import IDataHandler from freqtrade.exchange import Exchange, timeframe_to_minutes -from .datahandlers import get_datahandlerclass logger = logging.getLogger(__name__) @@ -106,8 +106,8 @@ def load_pair_history(pair: str, fill_up_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, - data_format: str = 'json', - data_handler: IDataHandler, + data_format: str = None, + data_handler: IDataHandler = None, ) -> DataFrame: """ Load cached ticker history for the given pair. @@ -115,18 +115,17 @@ def load_pair_history(pair: str, :param pair: Pair to load data for :param timeframe: Ticker timeframe (e.g. "5m") :param datadir: Path to the data storage location. + :param data_format: Format of the data. Ignored if data_handler is set. :param timerange: Limit data to be loaded to this timerange :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period - :param data_format: Format of the data. Ignored if data_handler is set. :param data_handler: Initialized data-handler to use. Will be initialized from data_format if not set :return: DataFrame with ohlcv data, or empty DataFrame """ - if not data_handler: - HandlerClass = get_datahandlerclass(data_format) - data_handler = HandlerClass(datadir) + data_handler = get_datahandler(datadir, data_format, data_handler) + return data_handler.ohlcv_load(pair=pair, timeframe=timeframe, timerange=timerange, @@ -138,7 +137,7 @@ def load_pair_history(pair: str, def load_data(datadir: Path, timeframe: str, - pairs: List[str], + pairs: List[str], *, timerange: Optional[TimeRange] = None, fill_up_missing: bool = True, startup_candles: int = 0, @@ -162,8 +161,7 @@ def load_data(datadir: Path, if startup_candles > 0 and timerange: logger.info(f'Using indicator startup period: {startup_candles} ...') - HandlerClass = get_datahandlerclass(data_format) - data_handler = HandlerClass(datadir) + data_handler = get_datahandler(datadir, data_format) for pair in pairs: hist = load_pair_history(pair=pair, timeframe=timeframe, @@ -184,6 +182,7 @@ def refresh_data(datadir: Path, timeframe: str, pairs: List[str], exchange: Exchange, + data_format: str = None, timerange: Optional[TimeRange] = None, ) -> None: """ @@ -195,10 +194,11 @@ def refresh_data(datadir: Path, :param exchange: Exchange object :param timerange: Limit data to be loaded to this timerange """ + data_handler = get_datahandler(datadir, data_format) for pair in pairs: _download_pair_history(pair=pair, timeframe=timeframe, datadir=datadir, timerange=timerange, - exchange=exchange) + exchange=exchange, data_handler=data_handler) def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path: @@ -256,9 +256,10 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str, def _download_pair_history(datadir: Path, exchange: Exchange, - pair: str, + pair: str, *, timeframe: str = '5m', - timerange: Optional[TimeRange] = None) -> bool: + timerange: Optional[TimeRange] = None, + data_handler: IDataHandler = None) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that @@ -272,6 +273,8 @@ def _download_pair_history(datadir: Path, :param timerange: range of time to download :return: bool with success state """ + data_handler = get_datahandler(datadir) + try: logger.info( f'Download history data for pair: "{pair}", timeframe: {timeframe} ' @@ -308,13 +311,14 @@ def _download_pair_history(datadir: Path, def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], datadir: Path, timerange: Optional[TimeRange] = None, - erase=False) -> List[str]: + erase=False, data_format: str = None) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. :return: List of pairs that are not available. """ pairs_not_available = [] + data_handler = get_datahandler(datadir, data_format) for pair in pairs: if pair not in exchange.markets: pairs_not_available.append(pair) @@ -331,7 +335,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes logger.info(f'Downloading pair {pair}, interval {timeframe}.') _download_pair_history(datadir=datadir, exchange=exchange, pair=pair, timeframe=str(timeframe), - timerange=timerange) + timerange=timerange, data_handler=data_handler) return pairs_not_available From 9876d126ca0add61c853a99a0a8a531445046dae Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 16:34:27 +0100 Subject: [PATCH 031/317] Use handler for trades --- freqtrade/data/converter.py | 9 ++++----- freqtrade/data/history.py | 38 ++++++++++++++++++------------------- tests/data/test_history.py | 7 ++++--- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 52ce3c593..09f7e3278 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -155,12 +155,12 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: return frame -def trades_to_ohlcv(trades: list, timeframe: str) -> list: +def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame: """ Converts trades list to ohlcv list :param trades: List of trades, as returned by ccxt.fetch_trades. :param timeframe: Ticker timeframe to resample data to - :return: ohlcv timeframe as list (as returned by ccxt.fetch_ohlcv) + :return: ohlcv Dataframe. """ from freqtrade.exchange import timeframe_to_minutes ticker_minutes = timeframe_to_minutes(timeframe) @@ -170,8 +170,7 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> list: df_new = df['price'].resample(f'{ticker_minutes}min').ohlc() df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum() - df_new['date'] = df_new.index.astype("int64") // 10 ** 6 + df_new['date'] = df_new.index # Drop 0 volume rows df_new = df_new.dropna() - columns = ["date", "open", "high", "low", "close", "volume"] - return list(zip(*[df_new[x].values.tolist() for x in columns])) + return df_new[['date', 'open', 'high', 'low', 'close', 'volume']] diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 4f31ebd60..1970f05d5 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -18,7 +18,7 @@ from pandas import DataFrame from freqtrade import OperationalException, misc from freqtrade.configuration import TimeRange from freqtrade.data.converter import trades_to_ohlcv -from freqtrade.data.datahandlers import get_datahandler, get_datahandlerclass +from freqtrade.data.datahandlers import get_datahandler from freqtrade.data.datahandlers.idatahandler import IDataHandler from freqtrade.exchange import Exchange, timeframe_to_minutes @@ -90,15 +90,6 @@ def load_trades_file(datadir: Path, pair: str, return tradesdata -def store_trades_file(datadir: Path, pair: str, - data: list, is_zip: bool = True): - """ - Stores tickerdata to file - """ - filename = pair_trades_filename(datadir, pair) - misc.file_dump_json(filename, data, is_zip=is_zip) - - def load_pair_history(pair: str, timeframe: str, datadir: Path, *, @@ -339,10 +330,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes return pairs_not_available -def _download_trades_history(datadir: Path, - exchange: Exchange, - pair: str, - timerange: Optional[TimeRange] = None) -> bool: +def _download_trades_history(exchange: Exchange, + pair: str, *, + timerange: Optional[TimeRange] = None, + data_handler: IDataHandler + ) -> bool: """ Download trade history from the exchange. Appends to previously downloaded trades data. @@ -351,7 +343,7 @@ def _download_trades_history(datadir: Path, since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None - trades = load_trades_file(datadir, pair) + trades = data_handler.trades_load(pair) from_id = trades[-1]['id'] if trades else None @@ -366,7 +358,7 @@ def _download_trades_history(datadir: Path, from_id=from_id, ) trades.extend(new_trades[1]) - store_trades_file(datadir, pair, trades) + data_handler.trades_store(pair, data=trades) logger.debug("New Start: %s", trades[0]['datetime']) logger.debug("New End: %s", trades[-1]['datetime']) @@ -382,13 +374,15 @@ def _download_trades_history(datadir: Path, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, - timerange: TimeRange, erase=False) -> List[str]: + timerange: TimeRange, erase=False, + data_format: str = 'jsongz') -> List[str]: """ Refresh stored trades data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. :return: List of pairs that are not available. """ pairs_not_available = [] + data_handler = get_datahandler(datadir, data_format=data_format) for pair in pairs: if pair not in exchange.markets: pairs_not_available.append(pair) @@ -404,7 +398,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: logger.info(f'Downloading trades for pair {pair}.') _download_trades_history(datadir=datadir, exchange=exchange, pair=pair, - timerange=timerange) + timerange=timerange, + data_handler=data_handler) return pairs_not_available @@ -413,8 +408,11 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], """ Convert stored trades data to ohlcv data """ + data_handler_trades = get_datahandler(datadir, data_format='jsongz') + data_handler_ohlcv = get_datahandler(datadir, data_format='json') + for pair in pairs: - trades = load_trades_file(datadir, pair) + trades = data_handler_trades.trades_load(pair) for timeframe in timeframes: ohlcv_file = pair_data_filename(datadir, pair, timeframe) if erase and ohlcv_file.exists(): @@ -422,7 +420,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], ohlcv_file.unlink() ohlcv = trades_to_ohlcv(trades, timeframe) # Store ohlcv - store_tickerdata_file(datadir, pair, timeframe, data=ohlcv) + data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv) def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 5f7d0f5a2..77cb6a565 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -10,6 +10,7 @@ import arrow from pandas import DataFrame from freqtrade.configuration import TimeRange +from freqtrade.data.datahandlers import get_datahandler from freqtrade.data.history import (_download_pair_history, _download_trades_history, _load_cached_data_for_updating, @@ -597,12 +598,12 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad ght_mock) exchange = get_patched_exchange(mocker, default_conf) file1 = testdatadir / 'ETH_BTC-trades.json.gz' - + data_handler = get_datahandler(testdatadir, data_format='jsongz') _backup_file(file1) assert not file1.is_file() - assert _download_trades_history(datadir=testdatadir, exchange=exchange, + assert _download_trades_history(data_handler=data_handler, exchange=exchange, pair='ETH/BTC') assert log_has("New Amount of trades: 5", caplog) assert file1.is_file() @@ -613,7 +614,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', MagicMock(side_effect=ValueError)) - assert not _download_trades_history(datadir=testdatadir, exchange=exchange, + assert not _download_trades_history(data_handler=data_handler, exchange=exchange, pair='ETH/BTC') assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) From b7c1d5549179ecd39f4e2c5b269b90df1e6a1be9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 16:41:52 +0100 Subject: [PATCH 032/317] Modify tests to point to datahandlers --- freqtrade/data/history.py | 20 -------------------- tests/data/test_history.py | 20 ++++++++++++++------ 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 1970f05d5..c83e91bac 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -76,20 +76,6 @@ def store_tickerdata_file(datadir: Path, pair: str, misc.file_dump_json(filename, data, is_zip=is_zip) -def load_trades_file(datadir: Path, pair: str, - timerange: Optional[TimeRange] = None) -> List[Dict]: - """ - Load a pair from file, either .json.gz or .json - :return: tradelist or empty list if unsuccesful - """ - filename = pair_trades_filename(datadir, pair) - tradesdata = misc.file_load_json(filename) - if not tradesdata: - return [] - - return tradesdata - - def load_pair_history(pair: str, timeframe: str, datadir: Path, *, @@ -198,12 +184,6 @@ def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path: return filename -def pair_trades_filename(datadir: Path, pair: str) -> Path: - pair_s = pair.replace("/", "_") - filename = datadir.joinpath(f'{pair_s}-trades.json.gz') - return filename - - def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 77cb6a565..fef7a3149 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -11,13 +11,14 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.data.datahandlers import get_datahandler +from freqtrade.data.datahandlers.jsondatahandler import (JsonDataHandler, + JsonGzDataHandler) from freqtrade.data.history import (_download_pair_history, _download_trades_history, _load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange, load_data, load_pair_history, - load_tickerdata_file, pair_data_filename, - pair_trades_filename, + load_tickerdata_file, refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data, trim_tickerlist, validate_backtest_data) @@ -143,14 +144,21 @@ def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) -def test_pair_data_filename(): - fn = pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m') +def test_json_pair_data_filename(): + fn = JsonDataHandler._pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m') assert isinstance(fn, Path) assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json') + fn = JsonGzDataHandler._pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m') + assert isinstance(fn, Path) + assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json.gz') -def test_pair_trades_filename(): - fn = pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC') +def test_json_pair_trades_filename(): + fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC') + assert isinstance(fn, Path) + assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json') + + fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC') assert isinstance(fn, Path) assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json.gz') From 552c93abf08239108b5b64baec0d37618152a3a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Dec 2019 19:53:52 +0100 Subject: [PATCH 033/317] Improve some docstrings --- .../data/datahandlers/jsondatahandler.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index c799784d0..0b6327017 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -20,7 +20,11 @@ class JsonDataHandler(IDataHandler): @classmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: """ - Returns a list of all pairs available in this datadir + Returns a list of all pairs with ohlcv data available in this datadir + for the specified timeframe + :param datadir: Directory to search for ohlcv files + :param timeframe: Timeframe to search pairs for + :return: List of Pairs """ _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name) @@ -56,7 +60,7 @@ class JsonDataHandler(IDataHandler): """ Internal method used to load data for one pair from disk. Implements the loading and conversation to a Pandas dataframe. - :param pair: Pair to load data for + :param pair: Pair to load data :param timeframe: Ticker timeframe (e.g. "5m") :param timerange: Limit data to be loaded to this timerange :param fill_missing: Fill missing values with "No action"-candles @@ -95,16 +99,20 @@ class JsonDataHandler(IDataHandler): @classmethod def trades_get_pairs(cls, datadir: Path) -> List[str]: """ - Returns a list of all pairs available in this datadir + Returns a list of all pairs for which trade data is available in this + :param datadir: Directory to search for ohlcv files + :return: List of Pairs """ _tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name) for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] # Check if regex found something and only return these results to avoid exceptions. return [match[0].replace('_', '/') for match in _tmp if match] - def trades_store(self, pair: str, data: List[Dict]): + def trades_store(self, pair: str, data: List[Dict]) -> None: """ - Store data + Store trades data (list of Dicts) to file + :param pair: Pair - used for filename + :param data: List of Dicts containing trade data """ filename = self._pair_trades_filename(self._datadir, pair) misc.file_dump_json(filename, data, is_zip=self._use_zip) @@ -112,13 +120,17 @@ class JsonDataHandler(IDataHandler): def trades_append(self, pair: str, data: DataFrame): """ Append data to existing files + :param pair: Pair - used for filename + :param data: List of Dicts containing trade data """ raise NotImplementedError() def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]: """ Load a pair from file, either .json.gz or .json - # TODO: validate timerange ... + # TODO: respect timerange ... + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented :return: List of trades """ filename = self._pair_trades_filename(self._datadir, pair) From e861f05b75fd333d21ab63b57cbd189e405ff27f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Dec 2019 09:51:03 +0100 Subject: [PATCH 034/317] Move dataframe trim to within jsondatahandler --- freqtrade/data/datahandlers/idatahandler.py | 6 +----- .../data/datahandlers/jsondatahandler.py | 19 ++++++++++++++++++- freqtrade/data/history.py | 8 +++----- tests/data/test_history.py | 5 +++-- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/datahandlers/idatahandler.py index c28d03930..8e61829b6 100644 --- a/freqtrade/data/datahandlers/idatahandler.py +++ b/freqtrade/data/datahandlers/idatahandler.py @@ -55,11 +55,7 @@ class IDataHandler(ABC): f'No history data for pair: "{pair}", timeframe: {timeframe}. ' 'Use `freqtrade download-data` to download the data' ) - return pairdf - else: - if timerange_startup: - self._validate_pairdata(pair, pairdf, timerange_startup) - return pairdf + return pairdf def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange): """ diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index 0b6327017..42fbaf51b 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -78,13 +78,18 @@ class JsonDataHandler(IDataHandler): utc=True, infer_datetime_format=True) + enddate = pairdata.iloc[-1]['date'] + if timerange: + self._validate_pairdata(pair, pairdata, timerange) pairdata = trim_dataframe(pairdata, timerange) + # incomplete candles should only be dropped if we didn't trim the end beforehand. return clean_ohlcv_dataframe(pairdata, timeframe, pair=pair, fill_missing=fill_missing, - drop_incomplete=drop_incomplete) + drop_incomplete=(drop_incomplete and + enddate == pairdata.iloc[-1]['date'])) def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: """ @@ -140,6 +145,18 @@ class JsonDataHandler(IDataHandler): return tradesdata + def trades_purge(self, pair: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_trades_filename(self._datadir, pair) + if filename.is_file(): + filename.unlink() + return True + return False + @classmethod def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: pair_s = pair.replace("/", "_") diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index c83e91bac..e71d9f36c 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -369,11 +369,9 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: logger.info(f"Skipping pair {pair}...") continue - dl_file = pair_trades_filename(datadir, pair) - if erase and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}.') - dl_file.unlink() + if erase: + if data_handler.trades_purge(pair): + logger.info(f'Deleting existing data for pair {pair}.') logger.info(f'Downloading trades for pair {pair}.') _download_trades_history(datadir=datadir, exchange=exchange, diff --git a/tests/data/test_history.py b/tests/data/test_history.py index fef7a3149..973c83a46 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -369,8 +369,9 @@ def test_load_partial_missing(testdatadir, caplog) -> None: # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(tickerdata['UNITTEST/BTC']) - # This validation happens now after parsing to pandas. - end_real = arrow.get(tickerdata['UNITTEST/BTC'].iloc[-1, 0]) + + # Shift endtime with +5 - as last candle is dropped (partial candle) + end_real = arrow.get(tickerdata['UNITTEST/BTC'].iloc[-1, 0]).shift(minutes=5) assert log_has(f'Missing data at end for pair ' f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}', caplog) From 37c5b689877958fbf80478e792e50b77d0183148 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Dec 2019 09:56:42 +0100 Subject: [PATCH 035/317] Move dataframe validation to abstract class --- freqtrade/data/datahandlers/idatahandler.py | 20 ++++++++++++---- .../data/datahandlers/jsondatahandler.py | 23 +++---------------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/datahandlers/idatahandler.py index 8e61829b6..ebf399358 100644 --- a/freqtrade/data/datahandlers/idatahandler.py +++ b/freqtrade/data/datahandlers/idatahandler.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Dict, List, Optional from datetime import datetime, timezone from pandas import DataFrame +from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe from freqtrade.configuration import TimeRange from freqtrade.exchange import timeframe_to_seconds @@ -47,15 +48,26 @@ class IDataHandler(ABC): timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) pairdf = self._ohlcv_load(pair, timeframe, - timerange=timerange_startup, - fill_missing=fill_missing, - drop_incomplete=drop_incomplete) + timerange=timerange_startup) if pairdf.empty: logger.warning( f'No history data for pair: "{pair}", timeframe: {timeframe}. ' 'Use `freqtrade download-data` to download the data' ) - return pairdf + return pairdf + else: + enddate = pairdf.iloc[-1]['date'] + + if timerange_startup: + self._validate_pairdata(pair, pairdf, timerange_startup) + pairdf = trim_dataframe(pairdf, timerange_startup) + + # incomplete candles should only be dropped if we didn't trim the end beforehand. + return clean_ohlcv_dataframe(pairdf, timeframe, + pair=pair, + fill_missing=fill_missing, + drop_incomplete=(drop_incomplete and + enddate == pairdf.iloc[-1]['date'])) def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange): """ diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index 42fbaf51b..7d780a48d 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -7,7 +7,6 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange -from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe from .idatahandler import IDataHandler @@ -54,18 +53,14 @@ class JsonDataHandler(IDataHandler): def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, - fill_missing: bool = True, - drop_incomplete: bool = True, ) -> DataFrame: """ Internal method used to load data for one pair from disk. Implements the loading and conversation to a Pandas dataframe. + Timerange trimming and dataframe validation happens outside of this method. :param pair: Pair to load data :param timeframe: Ticker timeframe (e.g. "5m") - :param timerange: Limit data to be loaded to this timerange - :param fill_missing: Fill missing values with "No action"-candles - :param drop_incomplete: Drop last candle assuming it may be incomplete. - :param startup_candles: Additional candles to load at the start of the period + :param timerange: Limit data to be loaded to this timerange. :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename(self._datadir, pair, timeframe) @@ -77,19 +72,7 @@ class JsonDataHandler(IDataHandler): unit='ms', utc=True, infer_datetime_format=True) - - enddate = pairdata.iloc[-1]['date'] - - if timerange: - self._validate_pairdata(pair, pairdata, timerange) - pairdata = trim_dataframe(pairdata, timerange) - - # incomplete candles should only be dropped if we didn't trim the end beforehand. - return clean_ohlcv_dataframe(pairdata, timeframe, - pair=pair, - fill_missing=fill_missing, - drop_incomplete=(drop_incomplete and - enddate == pairdata.iloc[-1]['date'])) + return pairdata def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: """ From 91c70a0e9c3d4299973fca60a3a922bf6f275307 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Dec 2019 10:22:38 +0100 Subject: [PATCH 036/317] Change to use ohlcv_purge --- freqtrade/data/datahandlers/jsondatahandler.py | 13 +++++++++++++ freqtrade/data/history.py | 9 ++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index 7d780a48d..e76fee2b9 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -74,6 +74,19 @@ class JsonDataHandler(IDataHandler): infer_datetime_format=True) return pairdata + def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :param timeframe: Ticker timeframe (e.g. "5m") + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + if filename.is_file(): + filename.unlink() + return True + return False + def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: """ Append data to existing data structures diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index e71d9f36c..9347e7ee5 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -374,7 +374,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: logger.info(f'Deleting existing data for pair {pair}.') logger.info(f'Downloading trades for pair {pair}.') - _download_trades_history(datadir=datadir, exchange=exchange, + _download_trades_history(exchange=exchange, pair=pair, timerange=timerange, data_handler=data_handler) @@ -392,10 +392,9 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], for pair in pairs: trades = data_handler_trades.trades_load(pair) for timeframe in timeframes: - ohlcv_file = pair_data_filename(datadir, pair, timeframe) - if erase and ohlcv_file.exists(): - logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') - ohlcv_file.unlink() + if erase: + if data_handler_ohlcv.ohlcv_purge(pair, timeframe): + logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') ohlcv = trades_to_ohlcv(trades, timeframe) # Store ohlcv data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv) From dbe8f727cb1e58b3c5139e18e5df05cfe397d760 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Dec 2019 10:26:19 +0100 Subject: [PATCH 037/317] Fix typehint --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4f2e990d2..74097286a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -434,7 +434,7 @@ class IStrategy(ABC): else: return current_profit > roi - def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: + def tickerdata_to_dataframe(self, tickerdata: Dict[str, DataFrame]) -> Dict[str, DataFrame]: """ Creates a dataframe and populates indicators for given ticker data Used by optimize operations only, not during dry / live runs. From b83487a70d32d71b3fea8998f619bfd984383887 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Dec 2019 19:52:08 +0100 Subject: [PATCH 038/317] Extract default dataframe columns to constant --- freqtrade/constants.py | 1 + freqtrade/data/converter.py | 5 +++-- freqtrade/data/datahandlers/jsondatahandler.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9c7e3db63..8d9cde98b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -21,6 +21,7 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'P AVAILABLE_DATAHANDLERS = ['json', 'jsongz'] DRY_RUN_WALLET = 1000 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons +DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume'] USERPATH_HYPEROPTS = 'hyperopts' USERPATH_STRATEGY = 'strategies' diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 09f7e3278..41a843e36 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -8,6 +8,7 @@ import pandas as pd from pandas import DataFrame, to_datetime from freqtrade.configuration.timerange import TimeRange +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS logger = logging.getLogger(__name__) @@ -26,7 +27,7 @@ def parse_ticker_dataframe(ticker: list, timeframe: str, pair: str, *, :return: DataFrame """ logger.debug("Parsing tickerlist to dataframe") - cols = ['date', 'open', 'high', 'low', 'close', 'volume'] + cols = DEFAULT_DATAFRAME_COLUMNS frame = DataFrame(ticker, columns=cols) frame['date'] = to_datetime(frame['date'], @@ -173,4 +174,4 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame: df_new['date'] = df_new.index # Drop 0 volume rows df_new = df_new.dropna() - return df_new[['date', 'open', 'high', 'low', 'close', 'volume']] + return df_new[DEFAULT_DATAFRAME_COLUMNS] diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index e76fee2b9..1a26671b9 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -7,6 +7,7 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from .idatahandler import IDataHandler @@ -14,7 +15,7 @@ from .idatahandler import IDataHandler class JsonDataHandler(IDataHandler): _use_zip = False - _columns = ['date', 'open', 'high', 'low', 'close', 'volume'] + _columns = DEFAULT_DATAFRAME_COLUMNS @classmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: From ec8fb5f30894ef55fc388e7c6767e17d3c24d57b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 06:58:29 +0100 Subject: [PATCH 039/317] Make no-data warning optional --- freqtrade/data/datahandlers/idatahandler.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/datahandlers/idatahandler.py index ebf399358..b80b8cfcc 100644 --- a/freqtrade/data/datahandlers/idatahandler.py +++ b/freqtrade/data/datahandlers/idatahandler.py @@ -30,6 +30,7 @@ class IDataHandler(ABC): fill_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, + warn_no_data: bool = True ) -> DataFrame: """ Load cached ticker history for the given pair. @@ -40,6 +41,7 @@ class IDataHandler(ABC): :param fill_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period + :param warn_no_data: Log a warning message when no data is found :return: DataFrame with ohlcv data, or empty DataFrame """ # Fix startup period @@ -50,10 +52,11 @@ class IDataHandler(ABC): pairdf = self._ohlcv_load(pair, timeframe, timerange=timerange_startup) if pairdf.empty: - logger.warning( - f'No history data for pair: "{pair}", timeframe: {timeframe}. ' - 'Use `freqtrade download-data` to download the data' - ) + if warn_no_data: + logger.warning( + f'No history data for pair: "{pair}", timeframe: {timeframe}. ' + 'Use `freqtrade download-data` to download the data' + ) return pairdf else: enddate = pairdf.iloc[-1]['date'] From c648d973c18015270d9196c3d9abc73eed09b13b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 06:58:50 +0100 Subject: [PATCH 040/317] Implement new "load_data_for_updating" method based on dataframes --- freqtrade/data/history.py | 58 +++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 9347e7ee5..80a846e87 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -8,7 +8,7 @@ Includes: import logging import operator -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -17,7 +17,8 @@ from pandas import DataFrame from freqtrade import OperationalException, misc from freqtrade.configuration import TimeRange -from freqtrade.data.converter import trades_to_ohlcv +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv from freqtrade.data.datahandlers import get_datahandler from freqtrade.data.datahandlers.idatahandler import IDataHandler from freqtrade.exchange import Exchange, timeframe_to_minutes @@ -184,9 +185,9 @@ def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path: return filename -def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str, - timerange: Optional[TimeRange]) -> Tuple[List[Any], - Optional[int]]: +def _load_cached_data_for_updating_old(datadir: Path, pair: str, timeframe: str, + timerange: Optional[TimeRange]) -> Tuple[List[Any], + Optional[int]]: """ Load cached data to download more data. If timerange is passed in, checks whether data from an before the stored data will be @@ -225,6 +226,27 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str, return (data, since_ms) +def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], + data_handler: IDataHandler) -> Tuple[DataFrame, Optional[int]]: + start = None + if timerange: + if timerange.starttype == 'date': + # TODO: convert to date for conversation + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + + # Intentionally don't pass timerange in - since we need to load the full dataset. + data = data_handler.ohlcv_load(pair, timeframe=timeframe, + timerange=None, fill_missing=False, + drop_incomplete=True, warn_no_data=False) + if not data.empty: + if start < data.iloc[0]['date']: + # Earlier data than existing data requested, redownload all + return DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS), None + start = data.iloc[-1]['date'] + start_ms = int(start.timestamp() * 1000) if start else None + return data, start_ms + + def _download_pair_history(datadir: Path, exchange: Exchange, pair: str, *, @@ -252,10 +274,14 @@ def _download_pair_history(datadir: Path, f'and store in {datadir}.' ) - data, since_ms = _load_cached_data_for_updating(datadir, pair, timeframe, timerange) + # data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange) + data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange, + data_handler=data_handler) - logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') - logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') + logger.debug("Current Start: %s", + f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') + logger.debug("Current End: %s", + f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_historic_ohlcv(pair=pair, @@ -264,12 +290,20 @@ def _download_pair_history(datadir: Path, int(arrow.utcnow().shift( days=-30).float_timestamp) * 1000 ) - data.extend(new_data) + # TODO: Maybe move parsing to exchange class (?) + new_dataframe = parse_ticker_dataframe(new_data, timeframe, pair, + fill_missing=False, drop_incomplete=True) + if data.empty: + data = new_dataframe + else: + data = data.append(new_dataframe) - logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) - logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) + logger.debug("New Start: %s", + f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') + logger.debug("New End: %s", + f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') - store_tickerdata_file(datadir, pair, timeframe, data=data) + data_handler.ohlcv_store(pair, timeframe, data=data) return True except Exception as e: From df085a6f1560d00f90dcbc69e16ff4ce9e021473 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 07:07:27 +0100 Subject: [PATCH 041/317] Fix small bug and test --- freqtrade/data/history.py | 4 +++- tests/data/test_history.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 80a846e87..189f51594 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -239,10 +239,12 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona timerange=None, fill_missing=False, drop_incomplete=True, warn_no_data=False) if not data.empty: - if start < data.iloc[0]['date']: + if start and start < data.iloc[0]['date']: # Earlier data than existing data requested, redownload all return DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS), None + start = data.iloc[-1]['date'] + start_ms = int(start.timestamp() * 1000) if start else None return data, start_ms diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 973c83a46..c626e27e8 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -301,7 +301,9 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: [1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839], [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] - json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) + json_dump_mock = mocker.patch( + 'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler.ohlcv_store', + return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') From e4f185f3579fe60e497c3efc5a416f45c2ad0d67 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 07:08:50 +0100 Subject: [PATCH 042/317] Remove 'line' from load_cached_data tests Users are unable to use line anyway, it's only there for tests --- tests/data/test_history.py | 38 ------------------------------ tests/optimize/test_backtesting.py | 1 - 2 files changed, 39 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index c626e27e8..4281eee8c 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -171,7 +171,6 @@ def test_load_cached_data_for_updating(mocker) -> None: with open(test_filename, "rt") as file: test_data = json.load(file) - # change now time to test 'line' cases # now = last cached item + 1 hour now_ts = test_data[-1][0] / 1000 + 60 * 60 mocker.patch('arrow.utcnow', return_value=arrow.get(now_ts)) @@ -183,13 +182,6 @@ def test_load_cached_data_for_updating(mocker) -> None: assert data == [] assert start_ts == test_data[0][0] - 1000 - # same with 'line' timeframe - num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', - TimeRange(None, 'line', 0, -num_lines)) - assert data == [] - assert start_ts < test_data[0][0] - 1 - # timeframe starts in the center of the cached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) @@ -197,13 +189,6 @@ def test_load_cached_data_for_updating(mocker) -> None: assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] - # same with 'line' timeframe - num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] - # timeframe starts after the chached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) @@ -211,22 +196,6 @@ def test_load_cached_data_for_updating(mocker) -> None: assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] - # Try loading last 30 lines. - # Not supported by _load_cached_data_for_updating, we always need to get the full data. - num_lines = 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] - - # no timeframe is set - # should return the chached data w/o the last item - num_lines = 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] - # no datafile exist # should return timestamp start time timerange = TimeRange('date', None, now_ts - 10000, 0) @@ -234,13 +203,6 @@ def test_load_cached_data_for_updating(mocker) -> None: assert data == [] assert start_ts == (now_ts - 10000) * 1000 - # same with 'line' timeframe - num_lines = 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = _load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange) - assert data == [] - assert start_ts == (now_ts - num_lines * 60) * 1000 - # no datafile exist, no timeframe is set # should return an empty array and None data, start_ts = _load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', None) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 8a27c591f..c4670a331 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -342,7 +342,6 @@ def test_tickerdata_with_fee(default_conf, mocker, testdatadir) -> None: def test_tickerdata_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) - # timerange = TimeRange(None, 'line', 0, -100) timerange = TimeRange.parse_timerange('1510694220-1510700340') tick = history.load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", From 7a6476c9ba979efb37d73ec2856e15e01b9e7535 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:11:49 +0100 Subject: [PATCH 043/317] Update tests --- freqtrade/data/history.py | 6 +++--- tests/data/test_history.py | 38 ++++++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 189f51594..99dde67c1 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -241,9 +241,9 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona if not data.empty: if start and start < data.iloc[0]['date']: # Earlier data than existing data requested, redownload all - return DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS), None - - start = data.iloc[-1]['date'] + data = DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) + else: + start = data.iloc[-1]['date'] start_ms = int(start.timestamp() * 1000) if start else None return data, start_ms diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 4281eee8c..12c6d3630 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -8,8 +8,10 @@ from unittest.mock import MagicMock, PropertyMock import arrow from pandas import DataFrame +from pandas.testing import assert_frame_equal from freqtrade.configuration import TimeRange +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.datahandlers import get_datahandler from freqtrade.data.datahandlers.jsondatahandler import (JsonDataHandler, JsonGzDataHandler) @@ -163,14 +165,17 @@ def test_json_pair_trades_filename(): assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json.gz') -def test_load_cached_data_for_updating(mocker) -> None: - datadir = Path(__file__).parent.parent.joinpath('testdata') +def test_load_cached_data_for_updating(mocker, testdatadir) -> None: + + data_handler = get_datahandler(testdatadir, 'json') test_data = None - test_filename = datadir.joinpath('UNITTEST_BTC-1m.json') + test_filename = testdatadir.joinpath('UNITTEST_BTC-1m.json') with open(test_filename, "rt") as file: test_data = json.load(file) + test_data_df = parse_ticker_dataframe(test_data, '1m', 'UNITTEST/BTC', + fill_missing=False, drop_incomplete=False) # now = last cached item + 1 hour now_ts = test_data[-1][0] / 1000 + 60 * 60 mocker.patch('arrow.utcnow', return_value=arrow.get(now_ts)) @@ -178,35 +183,36 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts earlier than the cached data # should fully update data timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == [] + data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler) + assert data.empty assert start_ts == test_data[0][0] - 1000 # timeframe starts in the center of the cached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] + data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler) + + assert_frame_equal(data, test_data_df.iloc[:-1]) + assert test_data[-2][0] <= start_ts < test_data[-1][0] # timeframe starts after the chached data # should return the chached data w/o the last item - timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] + timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 100, 0) + data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler) + assert_frame_equal(data, test_data_df.iloc[:-1]) + assert test_data[-2][0] <= start_ts < test_data[-1][0] # no datafile exist # should return timestamp start time timerange = TimeRange('date', None, now_ts - 10000, 0) - data, start_ts = _load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange) - assert data == [] + data, start_ts = _load_cached_data_for_updating('NONEXIST/BTC', '1m', timerange, data_handler) + assert data.empty assert start_ts == (now_ts - 10000) * 1000 # no datafile exist, no timeframe is set # should return an empty array and None - data, start_ts = _load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', None) - assert data == [] + data, start_ts = _load_cached_data_for_updating('NONEXIST/BTC', '1m', None, data_handler) + assert data.empty assert start_ts is None From d06777b8ce4021d0e378765f0bb484ecff567b87 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:12:56 +0100 Subject: [PATCH 044/317] Remove old "load_cached_data" method --- freqtrade/data/history.py | 41 ++++----------------------------------- 1 file changed, 4 insertions(+), 37 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 99dde67c1..93fe635a2 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -185,49 +185,16 @@ def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path: return filename -def _load_cached_data_for_updating_old(datadir: Path, pair: str, timeframe: str, - timerange: Optional[TimeRange]) -> Tuple[List[Any], - Optional[int]]: +def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], + data_handler: IDataHandler) -> Tuple[DataFrame, Optional[int]]: """ Load cached data to download more data. If timerange is passed in, checks whether data from an before the stored data will be downloaded. If that's the case then what's available should be completely overwritten. - Only used by download_pair_history(). + Otherwise downloads always start at the end of the available data to avoid data gaps. + Note: Only used by download_pair_history(). """ - - since_ms = None - - # user sets timerange, so find the start time - if timerange: - if timerange.starttype == 'date': - since_ms = timerange.startts * 1000 - elif timerange.stoptype == 'line': - num_minutes = timerange.stopts * timeframe_to_minutes(timeframe) - since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 - - # read the cached file - # Intentionally don't pass timerange in - since we need to load the full dataset. - data = load_tickerdata_file(datadir, pair, timeframe) - # remove the last item, could be incomplete candle - if data: - data.pop() - else: - data = [] - - if data: - if since_ms and since_ms < data[0][0]: - # Earlier data than existing data requested, redownload all - data = [] - else: - # a part of the data was already downloaded, so download unexist data only - since_ms = data[-1][0] + 1 - - return (data, since_ms) - - -def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], - data_handler: IDataHandler) -> Tuple[DataFrame, Optional[int]]: start = None if timerange: if timerange.starttype == 'date': From d1b52809ac79370023a47db66ee6474ee3b5ca9d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:25:17 +0100 Subject: [PATCH 045/317] Cleanup history --- freqtrade/data/history.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 93fe635a2..59695da2e 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -10,7 +10,7 @@ import logging import operator from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple import arrow from pandas import DataFrame @@ -21,7 +21,7 @@ from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv from freqtrade.data.datahandlers import get_datahandler from freqtrade.data.datahandlers.idatahandler import IDataHandler -from freqtrade.exchange import Exchange, timeframe_to_minutes +from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) @@ -68,15 +68,6 @@ def load_tickerdata_file(datadir: Path, pair: str, timeframe: str, return pairdata -def store_tickerdata_file(datadir: Path, pair: str, - timeframe: str, data: list, is_zip: bool = False): - """ - Stores tickerdata to file - """ - filename = pair_data_filename(datadir, pair, timeframe) - misc.file_dump_json(filename, data, is_zip=is_zip) - - def load_pair_history(pair: str, timeframe: str, datadir: Path, *, From a2567bea645c2d124423c2a7d4ff71f6c21a4abf Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:25:27 +0100 Subject: [PATCH 046/317] Remove unnecessary mock --- tests/optimize/test_backtesting.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index c4670a331..3b7b18f52 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -116,14 +116,6 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: assert len(results) == num_results -def mocked_load_data(datadir, pairs=[], timeframe='0m', - timerange=None, *args, **kwargs): - tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) - pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC", - fill_missing=True)} - return pairdata - - # use for mock ccxt.fetch_ohlvc' def _load_pair_as_ticks(pair, tickfreq): ticks = history.load_tickerdata_file(None, timeframe=tickfreq, pair=pair) @@ -460,7 +452,6 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: def get_timerange(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.data.history.load_data', mocked_load_data) mocker.patch('freqtrade.data.history.get_timerange', get_timerange) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) From aa39f2160bf095b1edecabc3243b72eed952cb69 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:25:38 +0100 Subject: [PATCH 047/317] Use load_data instead of a sequence of calls in tests which don't test this --- tests/optimize/test_hyperopt.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index fb492be35..ddafabe71 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -10,8 +10,7 @@ from arrow import Arrow from filelock import Timeout from freqtrade import OperationalException -from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.data.history import load_tickerdata_file +from freqtrade.data.history import load_data from freqtrade.optimize import setup_configuration, start_hyperopt from freqtrade.optimize.default_hyperopt import DefaultHyperOpt from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss @@ -543,9 +542,7 @@ def test_has_space(hyperopt, spaces, expected_results): def test_populate_indicators(hyperopt, testdatadir) -> None: - tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", - fill_missing=True)} + tickerlist = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) dataframes = hyperopt.backtesting.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) @@ -557,9 +554,7 @@ def test_populate_indicators(hyperopt, testdatadir) -> None: def test_buy_strategy_generator(hyperopt, testdatadir) -> None: - tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", - fill_missing=True)} + tickerlist = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) dataframes = hyperopt.backtesting.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) From 80dbba12801fbf7f030600de18d88d30c70a3014 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:29:06 +0100 Subject: [PATCH 048/317] Remove unnecessary mocks --- tests/optimize/test_backtesting.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 3b7b18f52..a10ce5254 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -116,13 +116,6 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: assert len(results) == num_results -# use for mock ccxt.fetch_ohlvc' -def _load_pair_as_ticks(pair, tickfreq): - ticks = history.load_tickerdata_file(None, timeframe=tickfreq, pair=pair) - ticks = ticks[-201:] - return ticks - - # FIX: fixturize this? def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record=None): data = history.load_data(datadir=datadir, timeframe='1m', pairs=[pair]) @@ -795,13 +788,7 @@ def test_backtest_record(default_conf, fee, mocker): def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - async def load_pairs(pair, timeframe, since): - return _load_pair_as_ticks(pair, timeframe) - - api_mock = MagicMock() - api_mock.fetch_ohlcv = load_pairs - - patch_exchange(mocker, api_mock) + patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock()) patched_configuration_load_config_file(mocker, default_conf) @@ -840,12 +827,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - async def load_pairs(pair, timeframe, since): - return _load_pair_as_ticks(pair, timeframe) - api_mock = MagicMock() - api_mock.fetch_ohlcv = load_pairs - - patch_exchange(mocker, api_mock) + patch_exchange(mocker) backtestmock = MagicMock() mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) gen_table_mock = MagicMock() From 5479c6717833ffff66a165b835ff20863f44ad48 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:40:14 +0100 Subject: [PATCH 049/317] Clean up some codes which use list-based tests --- tests/optimize/test_backtesting.py | 65 +++++++++++------------------- 1 file changed, 24 insertions(+), 41 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index a10ce5254..447f0052b 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1,6 +1,5 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument -import math import random from pathlib import Path from unittest.mock import MagicMock @@ -14,7 +13,7 @@ from freqtrade import DependencyException, OperationalException, constants from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import evaluate_result_multi -from freqtrade.data.converter import parse_ticker_dataframe +from freqtrade.data.converter import clean_ohlcv_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange from freqtrade.optimize import setup_configuration, start_backtesting @@ -50,47 +49,33 @@ def trim_dictlist(dict_list, num): def load_data_test(what, testdatadir): timerange = TimeRange.parse_timerange('1510694220-1510700340') - pair = history.load_tickerdata_file(testdatadir, timeframe='1m', - pair='UNITTEST/BTC', timerange=timerange) - datalen = len(pair) + data = history.load_pair_history(pair='UNITTEST/BTC', datadir=testdatadir, + timeframe='1m', timerange=timerange, + drop_incomplete=False, + fill_up_missing=False) base = 0.001 if what == 'raise': - data = [ - [ - pair[x][0], # Keep old dates - x * base, # But replace O,H,L,C - x * base + 0.0001, - x * base - 0.0001, - x * base, - pair[x][5], # Keep old volume - ] for x in range(0, datalen) - ] + data.loc[:, 'open'] = data.index * base + data.loc[:, 'high'] = data.index * base + 0.0001 + data.loc[:, 'low'] = data.index * base - 0.0001 + data.loc[:, 'close'] = data.index * base + if what == 'lower': - data = [ - [ - pair[x][0], # Keep old dates - 1 - x * base, # But replace O,H,L,C - 1 - x * base + 0.0001, - 1 - x * base - 0.0001, - 1 - x * base, - pair[x][5] # Keep old volume - ] for x in range(0, datalen) - ] + data.loc[:, 'open'] = 1 - data.index * base + data.loc[:, 'high'] = 1 - data.index * base + 0.0001 + data.loc[:, 'low'] = 1 - data.index * base - 0.0001 + data.loc[:, 'close'] = 1 - data.index * base + if what == 'sine': hz = 0.1 # frequency - data = [ - [ - pair[x][0], # Keep old dates - math.sin(x * hz) / 1000 + base, # But replace O,H,L,C - math.sin(x * hz) / 1000 + base + 0.0001, - math.sin(x * hz) / 1000 + base - 0.0001, - math.sin(x * hz) / 1000 + base, - pair[x][5] # Keep old volume - ] for x in range(0, datalen) - ] - return {'UNITTEST/BTC': parse_ticker_dataframe(data, '1m', pair="UNITTEST/BTC", - fill_missing=True)} + data.loc[:, 'open'] = np.sin(data.index * hz) / 1000 + base + data.loc[:, 'high'] = np.sin(data.index * hz) / 1000 + base + 0.0001 + data.loc[:, 'low'] = np.sin(data.index * hz) / 1000 + base - 0.0001 + data.loc[:, 'close'] = np.sin(data.index * hz) / 1000 + base + + return {'UNITTEST/BTC': clean_ohlcv_dataframe(data, timeframe='1m', pair='UNITTEST/BTC', + fill_missing=True)} def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: @@ -328,10 +313,8 @@ def test_tickerdata_with_fee(default_conf, mocker, testdatadir) -> None: def test_tickerdata_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) timerange = TimeRange.parse_timerange('1510694220-1510700340') - tick = history.load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", - fill_missing=True)} - + tickerlist = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, + fill_up_missing=True) backtesting = Backtesting(default_conf) data = backtesting.strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 102 From 4b277afc52c3ffbcfd668deac649e39515f3b428 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:41:19 +0100 Subject: [PATCH 050/317] Remove test for load_tickerdata --- tests/data/test_history.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 12c6d3630..969e58223 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -20,7 +20,6 @@ from freqtrade.data.history import (_download_pair_history, _load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange, load_data, load_pair_history, - load_tickerdata_file, refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data, trim_tickerlist, validate_backtest_data) @@ -303,17 +302,6 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, ) -def test_load_tickerdata_file(testdatadir) -> None: - # 7 does not exist in either format. - assert not load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '7m') - # 1 exists only as a .json - tickerdata = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m') - assert _BTC_UNITTEST_LENGTH == len(tickerdata) - # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json - tickerdata = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '8m') - assert _BTC_UNITTEST_LENGTH == len(tickerdata) - - def test_load_partial_missing(testdatadir, caplog) -> None: # Make sure we start fresh - test missing data at start start = arrow.get('2018-01-01T00:00:00') From 32c2ce146ebcbea093e32f66de3bedfcd0b28f01 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:44:08 +0100 Subject: [PATCH 051/317] Remove last usage of load_tickerlist --- tests/strategy/test_interface.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 89c38bda1..647344653 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -7,8 +7,7 @@ import arrow from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.data.history import load_tickerdata_file +from freqtrade.data.history import load_data from freqtrade.persistence import Trade from tests.conftest import get_patched_exchange, log_has from freqtrade.strategy.default_strategy import DefaultStrategy @@ -107,9 +106,8 @@ def test_tickerdata_to_dataframe(default_conf, testdatadir) -> None: strategy = DefaultStrategy(default_conf) timerange = TimeRange.parse_timerange('1510694220-1510700340') - tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", - fill_missing=True)} + tickerlist = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, + fill_up_missing=True) data = strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 102 # partial candle was removed From baa942ff989adb86c35228454887cd624d38a6c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:49:01 +0100 Subject: [PATCH 052/317] Don't use function to resolve pairname for test --- tests/test_misc.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_misc.py b/tests/test_misc.py index 23231e2f0..c5bf06311 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -5,7 +5,6 @@ from pathlib import Path from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.data.history import pair_data_filename from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json, file_load_json, format_ms_time, plural, shorten_date) @@ -48,13 +47,13 @@ def test_file_dump_json(mocker) -> None: def test_file_load_json(mocker, testdatadir) -> None: # 7m .json does not exist - ret = file_load_json(pair_data_filename(testdatadir, 'UNITTEST/BTC', '7m')) + ret = file_load_json(testdatadir / 'UNITTEST_BTC-7m.json') assert not ret # 1m json exists (but no .gz exists) - ret = file_load_json(pair_data_filename(testdatadir, 'UNITTEST/BTC', '1m')) + ret = file_load_json(testdatadir / 'UNITTEST_BTC-1m.json') assert ret # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json - ret = file_load_json(pair_data_filename(testdatadir, 'UNITTEST/BTC', '8m')) + ret = file_load_json(testdatadir / 'UNITTEST_BTC-8m.json') assert ret From a3144cb2f0c07c777f566326f0df9246dfbc36d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:49:30 +0100 Subject: [PATCH 053/317] remove trim_tickerlist --- freqtrade/data/history.py | 59 ++++---------------------------------- tests/data/test_history.py | 54 +--------------------------------- 2 files changed, 6 insertions(+), 107 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 59695da2e..9dad90864 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -15,7 +15,7 @@ from typing import Dict, List, Optional, Tuple import arrow from pandas import DataFrame -from freqtrade import OperationalException, misc +from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv @@ -26,48 +26,6 @@ from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) -def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: - """ - Trim tickerlist based on given timerange - """ - if not tickerlist: - return tickerlist - - start_index = 0 - stop_index = len(tickerlist) - - if timerange.starttype == 'date': - while (start_index < len(tickerlist) and - tickerlist[start_index][0] < timerange.startts * 1000): - start_index += 1 - - if timerange.stoptype == 'date': - while (stop_index > 0 and - tickerlist[stop_index-1][0] > timerange.stopts * 1000): - stop_index -= 1 - - if start_index > stop_index: - raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') - - return tickerlist[start_index:stop_index] - - -def load_tickerdata_file(datadir: Path, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None) -> List[Dict]: - """ - Load a pair from file, either .json.gz or .json - :return: tickerlist or None if unsuccessful - """ - filename = pair_data_filename(datadir, pair, timeframe) - pairdata = misc.file_load_json(filename) - if not pairdata: - return [] - - if timerange: - pairdata = trim_tickerlist(pairdata, timerange) - return pairdata - - def load_pair_history(pair: str, timeframe: str, datadir: Path, *, @@ -170,12 +128,6 @@ def refresh_data(datadir: Path, exchange=exchange, data_handler=data_handler) -def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path: - pair_s = pair.replace("/", "_") - filename = datadir.joinpath(f'{pair_s}-{timeframe}.json') - return filename - - def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], data_handler: IDataHandler) -> Tuple[DataFrame, Optional[int]]: """ @@ -291,11 +243,10 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes continue for timeframe in timeframes: - dl_file = pair_data_filename(datadir, pair, timeframe) - if erase and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}, interval {timeframe}.') - dl_file.unlink() + if erase: + if data_handler.ohlcv_purge(pair, timeframe): + logger.info( + f'Deleting existing data for pair {pair}, interval {timeframe}.') logger.info(f'Downloading pair {pair}, interval {timeframe}.') _download_pair_history(datadir=datadir, exchange=exchange, diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 969e58223..7e030b523 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -22,7 +22,7 @@ from freqtrade.data.history import (_download_pair_history, load_data, load_pair_history, refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data, - trim_tickerlist, validate_backtest_data) + validate_backtest_data) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.strategy.default_strategy import DefaultStrategy @@ -358,58 +358,6 @@ def test_init_with_refresh(default_conf, mocker) -> None: ) -def test_trim_tickerlist(testdatadir) -> None: - file = testdatadir / 'UNITTEST_BTC-1m.json' - with open(file) as data_file: - ticker_list = json.load(data_file) - ticker_list_len = len(ticker_list) - - # Test the pattern ^(\d{8})-(\d{8})$ - # This pattern extract a window between the dates - timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 5 - assert ticker_list[0] is not ticker[0] # The first element should be different - assert ticker_list[5] is ticker[0] # The list starts at the index 5 - assert ticker_list[9] is ticker[-1] # The list ends at the index 9 (5 elements) - - # Test the pattern ^-(\d{8})$ - # This pattern extracts elements from the start to the date - timerange = TimeRange(None, 'date', 0, ticker_list[10][0] / 1000 - 1) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 10 - assert ticker_list[0] is ticker[0] # The start of the list is included - assert ticker_list[9] is ticker[-1] # The element 10 is not included - - # Test the pattern ^(\d{8})-$ - # This pattern extracts elements from the date to now - timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, None) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == ticker_list_len - 10 - assert ticker_list[10] is ticker[0] # The first element is element #10 - assert ticker_list[-1] is ticker[-1] # The last element is the same - - # Test a wrong pattern - # This pattern must return the list unchanged - timerange = TimeRange(None, None, None, 5) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_list_len == ticker_len - - # passing empty list - timerange = TimeRange(None, None, None, 5) - ticker = trim_tickerlist([], timerange) - assert 0 == len(ticker) - assert not ticker - - def test_file_dump_json_tofile(testdatadir) -> None: file = testdatadir / 'test_{id}.json'.format(id=str(uuid.uuid4())) data = {'bar': 'foo'} From 8a030e7fc01566191ce747767efb2a6a1b83974f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 11:08:47 +0100 Subject: [PATCH 054/317] Use exists instead of is_file --- freqtrade/data/datahandlers/jsondatahandler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index 1a26671b9..9f3468d30 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -65,7 +65,7 @@ class JsonDataHandler(IDataHandler): :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename(self._datadir, pair, timeframe) - if not filename.is_file(): + if not filename.exists(): return DataFrame(columns=self._columns) pairdata = read_json(filename, orient='values') pairdata.columns = self._columns @@ -149,7 +149,7 @@ class JsonDataHandler(IDataHandler): :return: True when deleted, false if file did not exist. """ filename = self._pair_trades_filename(self._datadir, pair) - if filename.is_file(): + if filename.exists(): filename.unlink() return True return False From d65c1eea7aff4b730771a8d1a7d8d98755e9ba86 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 13:16:53 +0100 Subject: [PATCH 055/317] Add some tests for datahandler --- .../data/datahandlers/jsondatahandler.py | 4 +- tests/data/test_history.py | 50 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/datahandlers/jsondatahandler.py index 9f3468d30..f40cf969f 100644 --- a/freqtrade/data/datahandlers/jsondatahandler.py +++ b/freqtrade/data/datahandlers/jsondatahandler.py @@ -83,7 +83,7 @@ class JsonDataHandler(IDataHandler): :return: True when deleted, false if file did not exist. """ filename = self._pair_data_filename(self._datadir, pair, timeframe) - if filename.is_file(): + if filename.exists(): filename.unlink() return True return False @@ -119,7 +119,7 @@ class JsonDataHandler(IDataHandler): filename = self._pair_trades_filename(self._datadir, pair) misc.file_dump_json(filename, data, is_zip=self._use_zip) - def trades_append(self, pair: str, data: DataFrame): + def trades_append(self, pair: str, data: List[Dict]): """ Append data to existing files :param pair: Pair - used for filename diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 7e030b523..8787a35b2 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -7,6 +7,7 @@ from shutil import copyfile from unittest.mock import MagicMock, PropertyMock import arrow +import pytest from pandas import DataFrame from pandas.testing import assert_frame_equal @@ -561,3 +562,52 @@ def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): _clean_test_file(file1) _clean_test_file(file5) + + +def test_jsondatahandler_ohlcv_get_pairs(testdatadir): + pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m') + # Convert to set to avoid failures due to sorting + assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC', + 'XMR/BTC', 'ZEC/BTC', 'ADA/BTC', 'ETC/BTC', 'NXT/BTC', + 'DASH/BTC', 'XRP/ETH'} + + pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m') + assert set(pairs) == {'UNITTEST/BTC'} + + +def test_jsondatahandler_trades_get_pairs(testdatadir): + pairs = JsonGzDataHandler.trades_get_pairs(testdatadir) + # Convert to set to avoid failures due to sorting + assert set(pairs) == {'XRP/ETH'} + + +def test_jsondatahandler_ohlcv_purge(mocker, testdatadir): + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + mocker.patch.object(Path, "unlink", MagicMock()) + dh = JsonGzDataHandler(testdatadir) + assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + + +def test_jsondatahandler_trades_purge(mocker, testdatadir): + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + mocker.patch.object(Path, "unlink", MagicMock()) + dh = JsonGzDataHandler(testdatadir) + assert not dh.trades_purge('UNITTEST/NONEXIST') + + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + assert dh.trades_purge('UNITTEST/NONEXIST') + + +def test_jsondatahandler_ohlcv_append(testdatadir): + dh = JsonGzDataHandler(testdatadir) + with pytest.raises(NotImplementedError): + dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame()) + + +def test_jsondatahandler_trades_append(testdatadir): + dh = JsonGzDataHandler(testdatadir) + with pytest.raises(NotImplementedError): + dh.trades_append('UNITTEST/ETH', []) From 65f539e9d8bb0631237c6dc3c44cf42eb8d94e87 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 13:25:34 +0100 Subject: [PATCH 056/317] More tests for datahandler --- tests/data/test_history.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 8787a35b2..726b6d3a7 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -13,7 +13,8 @@ from pandas.testing import assert_frame_equal from freqtrade.configuration import TimeRange from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.data.datahandlers import get_datahandler +from freqtrade.data.datahandlers import get_datahandler, get_datahandlerclass +from freqtrade.data.datahandlers.idatahandler import IDataHandler from freqtrade.data.datahandlers.jsondatahandler import (JsonDataHandler, JsonGzDataHandler) from freqtrade.data.history import (_download_pair_history, @@ -611,3 +612,24 @@ def test_jsondatahandler_trades_append(testdatadir): dh = JsonGzDataHandler(testdatadir) with pytest.raises(NotImplementedError): dh.trades_append('UNITTEST/ETH', []) + + +def test_gethandlerclass(): + cl = get_datahandlerclass('json') + assert cl == JsonDataHandler + assert issubclass(cl, IDataHandler) + cl = get_datahandlerclass('jsongz') + assert cl == JsonGzDataHandler + assert issubclass(cl, IDataHandler) + assert issubclass(cl, JsonDataHandler) + with pytest.raises(ValueError, match=r"No datahandler for .*"): + get_datahandlerclass('DeadBeef') + + +def test_get_datahandler(testdatadir): + dh = get_datahandler(testdatadir, 'json') + assert type(dh) == JsonDataHandler + dh = get_datahandler(testdatadir, 'jsongz') + assert type(dh) == JsonGzDataHandler + dh1 = get_datahandler(testdatadir, 'jsongz', dh) + assert id(dh1) == id(dh) From 9c5b94adf506f2e726b2e497655a424c2cc6f4f5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 13:46:25 +0100 Subject: [PATCH 057/317] Pass data_format to methods --- docs/configuration.md | 2 ++ freqtrade/configuration/arguments.py | 2 +- freqtrade/configuration/cli_options.py | 12 ++++++++++++ freqtrade/configuration/configuration.py | 7 +++++++ freqtrade/constants.py | 14 +++++++------- freqtrade/data/history.py | 10 ++++++---- freqtrade/utils.py | 12 +++++++++--- 7 files changed, 44 insertions(+), 15 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 90f0aa791..002e3ab5d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -106,6 +106,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
***Datatype:*** *Boolean* | `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
***Datatype:*** *String* | `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
***Datatype:*** *String* +| `dataformat_ohlcv` | Data format to use to store OHLCV historic data.
*Defaults to `json`*.
***Datatype:*** *String* +| `dataformat_trades` | Data format to use to store trades historic data.
*Defaults to `jsongz`*.
***Datatype:*** *String* ### Parameters in the strategy diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index cefa86927..121bd23b4 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -51,7 +51,7 @@ ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", - "timeframes", "erase"] + "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index ffaf9ea66..c80bea393 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -344,6 +344,18 @@ AVAILABLE_CLI_OPTIONS = { choices=constants.AVAILABLE_DATAHANDLERS, required=True, ), + "dataformat_ohlcv": Arg( + '--data-format', + help='Storage format for downloaded ohlcv data. (default: `%(default)s`).', + choices=constants.AVAILABLE_DATAHANDLERS, + default='json' + ), + "dataformat_trades": Arg( + '--data-format-trades', + help='Storage format for downloaded trades data. (default: `%(default)s`).', + choices=constants.AVAILABLE_DATAHANDLERS, + default='jsongz' + ), "exchange": Arg( '--exchange', help=f'Exchange name (default: `{constants.DEFAULT_EXCHANGE}`). ' diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index f73b52c10..aa453a392 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -339,9 +339,16 @@ class Configuration: self._args_to_config(config, argname='days', logstring='Detected --days: {}') + self._args_to_config(config, argname='download_trades', logstring='Detected --dl-trades: {}') + self._args_to_config(config, argname='dataformat_ohlcv', + logstring='Using "{}" to store OHLCV data.') + + self._args_to_config(config, argname='dataformat_trade', + logstring='Using "{}" to store trades data.') + def _process_runmode(self, config: Dict[str, Any]) -> None: if not self.runmode: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 8d9cde98b..95ec71552 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -216,17 +216,17 @@ CONF_SCHEMA = { 'process_throttle_secs': {'type': 'integer'}, 'interval': {'type': 'integer'}, 'sd_notify': {'type': 'boolean'}, - 'dataformat_ohlcv': { - 'type': 'string', + } + }, + 'dataformat_ohlcv': { + 'type': 'string', 'enum': AVAILABLE_DATAHANDLERS, 'default': 'json' - }, - 'dataformat_trades': { - 'type': 'string', + }, + 'dataformat_trades': { + 'type': 'string', 'enum': AVAILABLE_DATAHANDLERS, 'default': 'jsongz' - } - } } }, 'definitions': { diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 9dad90864..c06b14b4a 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -178,7 +178,7 @@ def _download_pair_history(datadir: Path, :param timerange: range of time to download :return: bool with success state """ - data_handler = get_datahandler(datadir) + data_handler = get_datahandler(datadir, data_handler=data_handler) try: logger.info( @@ -327,12 +327,14 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], - datadir: Path, timerange: TimeRange, erase=False) -> None: + datadir: Path, timerange: TimeRange, erase=False, + data_format_ohlcv: str = 'json', + data_format_trades: str = 'jsongz') -> None: """ Convert stored trades data to ohlcv data """ - data_handler_trades = get_datahandler(datadir, data_format='jsongz') - data_handler_ohlcv = get_datahandler(datadir, data_format='json') + data_handler_trades = get_datahandler(datadir, data_format=data_format_trades) + data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv) for pair in pairs: trades = data_handler_trades.trades_load(pair) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 5f646cc7a..81d41f28a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -204,16 +204,22 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( exchange, pairs=config["pairs"], datadir=config['datadir'], - timerange=timerange, erase=config.get("erase")) + timerange=timerange, erase=config.get("erase"), + data_format=config['dataformat_trades']) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) + datadir=config['datadir'], timerange=timerange, erase=config.get("erase"), + data_format_ohlcv=config['dataformat_ohlcv'], + data_format_trades=config['dataformat_trades'], + ) + else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) + datadir=config['datadir'], timerange=timerange, erase=config.get("erase"), + data_format=config['dataformat_ohlcv']) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") From b37b5c3d90440d8d3afe098f8668d463a67c33c2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 07:02:46 +0100 Subject: [PATCH 058/317] Remove Explicit datadir conversation --- freqtrade/utils.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 81d41f28a..f5865f1db 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -251,16 +251,15 @@ def start_list_strategies(args: Dict[str, Any]) -> None: def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str): """ TODO: move this to converter.py (?) - TODO: remove Path conversation once PR is merged and this is rebased """ SrcClass = get_datahandlerclass(convert_from) TrgClass = get_datahandlerclass(convert_to) if 'pairs' not in config: - config['pairs'] = SrcClass.trades_get_pairs(Path(config['datadir'])) + config['pairs'] = SrcClass.trades_get_pairs(config['datadir']) logger.info(f"Converting trades for {config['pairs']}") - src = SrcClass(Path(config['datadir'])) - trg = TrgClass(Path(config['datadir'])) + src = SrcClass(config['datadir']) + trg = TrgClass(config['datadir']) for pair in config['pairs']: data = src.trades_load(pair=pair) logger.info(f"Converting {len(data)} trades for {pair}") @@ -270,7 +269,6 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str): """ TODO: move this to converter.py (?) - TODO: remove Path conversation once PR is merged and this is rebased """ SrcClass = get_datahandlerclass(convert_from) TrgClass = get_datahandlerclass(convert_to) @@ -281,12 +279,12 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: config['pairs'] = [] # Check timeframes or fall back to ticker_interval. for timeframe in timeframes: - config['pairs'].extend(SrcClass.ohlcv_get_pairs(Path(config['datadir']), + config['pairs'].extend(SrcClass.ohlcv_get_pairs(config['datadir'], timeframe)) logger.info(f"Converting OHLCV for {config['pairs']}") - src = SrcClass(Path(config['datadir'])) - trg = TrgClass(Path(config['datadir'])) + src = SrcClass(config['datadir']) + trg = TrgClass(config['datadir']) for timeframe in timeframes: for pair in config['pairs']: From 68604911891d7f0f9eaf2d9f2636f2110cc806c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 09:59:47 +0100 Subject: [PATCH 059/317] Rename datahandler module to history module Also move previous history.py into this module - so everything is bundled --- freqtrade/data/datahandlers/__init__.py | 36 ----------------- freqtrade/data/history/__init__.py | 14 +++++++ .../{history.py => history/history_utils.py} | 11 +---- .../{datahandlers => history}/idatahandler.py | 40 +++++++++++++++++-- .../jsondatahandler.py | 0 freqtrade/utils.py | 2 +- tests/data/test_history.py | 33 ++++++++------- tests/optimize/test_backtesting.py | 3 +- 8 files changed, 71 insertions(+), 68 deletions(-) delete mode 100644 freqtrade/data/datahandlers/__init__.py create mode 100644 freqtrade/data/history/__init__.py rename freqtrade/data/{history.py => history/history_utils.py} (98%) rename freqtrade/data/{datahandlers => history}/idatahandler.py (76%) rename freqtrade/data/{datahandlers => history}/jsondatahandler.py (100%) diff --git a/freqtrade/data/datahandlers/__init__.py b/freqtrade/data/datahandlers/__init__.py deleted file mode 100644 index a21dd832d..000000000 --- a/freqtrade/data/datahandlers/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -from typing import Type -from pathlib import Path -from .idatahandler import IDataHandler - - -def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: - """ - Get datahandler class. - Could be done using Resolvers, but since this may be called often and resolvers - are rather expensive, doing this directly should improve performance. - :param datatype: datatype to use. - :return: Datahandler class - """ - - if datatype == 'json': - from .jsondatahandler import JsonDataHandler - return JsonDataHandler - elif datatype == 'jsongz': - from .jsondatahandler import JsonGzDataHandler - return JsonGzDataHandler - else: - raise ValueError(f"No datahandler for datatype {datatype} available.") - - -def get_datahandler(datadir: Path, data_format: str = None, - data_handler: IDataHandler = None) -> IDataHandler: - """ - :param datadir: Folder to save data - :data_format: dataformat to use - :data_handler: returns this datahandler if it exists or initializes a new one - """ - - if not data_handler: - HandlerClass = get_datahandlerclass(data_format or 'json') - data_handler = HandlerClass(datadir) - return data_handler diff --git a/freqtrade/data/history/__init__.py b/freqtrade/data/history/__init__.py new file mode 100644 index 000000000..572c063fc --- /dev/null +++ b/freqtrade/data/history/__init__.py @@ -0,0 +1,14 @@ +""" +Handle historic data (ohlcv). + +Includes: +* load data for a pair (or a list of pairs) from disk +* download data from exchange and store to disk +""" + +from .history_utils import (convert_trades_to_ohlcv, # noqa: F401 + get_timerange, load_data, load_pair_history, + refresh_backtest_ohlcv_data, + refresh_backtest_trades_data, refresh_data, + validate_backtest_data) +from .idatahandler import get_datahandler, get_datahandlerclass # noqa: F401 diff --git a/freqtrade/data/history.py b/freqtrade/data/history/history_utils.py similarity index 98% rename from freqtrade/data/history.py rename to freqtrade/data/history/history_utils.py index c06b14b4a..42f1c9be7 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history/history_utils.py @@ -1,11 +1,3 @@ -""" -Handle historic data (ohlcv). - -Includes: -* load data for a pair (or a list of pairs) from disk -* download data from exchange and store to disk -""" - import logging import operator from datetime import datetime, timezone @@ -19,8 +11,7 @@ from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv -from freqtrade.data.datahandlers import get_datahandler -from freqtrade.data.datahandlers.idatahandler import IDataHandler +from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) diff --git a/freqtrade/data/datahandlers/idatahandler.py b/freqtrade/data/history/idatahandler.py similarity index 76% rename from freqtrade/data/datahandlers/idatahandler.py rename to freqtrade/data/history/idatahandler.py index b80b8cfcc..cee43dcef 100644 --- a/freqtrade/data/datahandlers/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -6,13 +6,14 @@ It's subclasses handle and storing data from disk. import logging from abc import ABC, abstractclassmethod, abstractmethod from copy import deepcopy -from pathlib import Path -from typing import Dict, List, Optional from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, List, Optional, Type + from pandas import DataFrame -from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe from freqtrade.configuration import TimeRange +from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe from freqtrade.exchange import timeframe_to_seconds logger = logging.getLogger(__name__) @@ -89,3 +90,36 @@ class IDataHandler(ABC): if pairdata.iloc[-1]['date'] < stop: logger.warning(f"Missing data at end for pair {pair}, " f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}") + + +def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: + """ + Get datahandler class. + Could be done using Resolvers, but since this may be called often and resolvers + are rather expensive, doing this directly should improve performance. + :param datatype: datatype to use. + :return: Datahandler class + """ + + if datatype == 'json': + from .jsondatahandler import JsonDataHandler + return JsonDataHandler + elif datatype == 'jsongz': + from .jsondatahandler import JsonGzDataHandler + return JsonGzDataHandler + else: + raise ValueError(f"No datahandler for datatype {datatype} available.") + + +def get_datahandler(datadir: Path, data_format: str = None, + data_handler: IDataHandler = None) -> IDataHandler: + """ + :param datadir: Folder to save data + :data_format: dataformat to use + :data_handler: returns this datahandler if it exists or initializes a new one + """ + + if not data_handler: + HandlerClass = get_datahandlerclass(data_format or 'json') + data_handler = HandlerClass(datadir) + return data_handler diff --git a/freqtrade/data/datahandlers/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py similarity index 100% rename from freqtrade/data/datahandlers/jsondatahandler.py rename to freqtrade/data/history/jsondatahandler.py diff --git a/freqtrade/utils.py b/freqtrade/utils.py index f5865f1db..1bb0f611a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -17,8 +17,8 @@ from freqtrade.configuration import (Configuration, TimeRange, from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY -from freqtrade.data.datahandlers import get_datahandlerclass from freqtrade.data.history import (convert_trades_to_ohlcv, + get_datahandlerclass, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) from freqtrade.exchange import (available_exchanges, ccxt_exchanges, diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 726b6d3a7..f84e819ad 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -13,18 +13,15 @@ from pandas.testing import assert_frame_equal from freqtrade.configuration import TimeRange from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.data.datahandlers import get_datahandler, get_datahandlerclass -from freqtrade.data.datahandlers.idatahandler import IDataHandler -from freqtrade.data.datahandlers.jsondatahandler import (JsonDataHandler, - JsonGzDataHandler) -from freqtrade.data.history import (_download_pair_history, - _download_trades_history, - _load_cached_data_for_updating, - convert_trades_to_ohlcv, get_timerange, - load_data, load_pair_history, - refresh_backtest_ohlcv_data, - refresh_backtest_trades_data, refresh_data, - validate_backtest_data) +from freqtrade.data.history import get_datahandler, get_datahandlerclass +from freqtrade.data.history.history_utils import ( + _download_pair_history, _download_trades_history, + _load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange, + load_data, load_pair_history, refresh_backtest_ohlcv_data, + refresh_backtest_trades_data, refresh_data, validate_backtest_data) +from freqtrade.data.history.idatahandler import IDataHandler +from freqtrade.data.history.jsondatahandler import (JsonDataHandler, + JsonGzDataHandler) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.strategy.default_strategy import DefaultStrategy @@ -100,7 +97,7 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: ltfmock = mocker.patch( - 'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler._ohlcv_load', + 'freqtrade.data.history.jsondatahandler.JsonDataHandler._ohlcv_load', MagicMock(return_value=DataFrame())) timerange = TimeRange('date', None, 1510639620, 0) load_pair_history(pair='UNITTEST/BTC', timeframe='1m', @@ -271,7 +268,7 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch( - 'freqtrade.data.datahandlers.jsondatahandler.JsonDataHandler.ohlcv_store', + 'freqtrade.data.history.jsondatahandler.JsonDataHandler.ohlcv_store', return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) @@ -444,7 +441,8 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', + MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) @@ -465,7 +463,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', MagicMock()) ex = get_patched_exchange(mocker, default_conf) mocker.patch( @@ -485,7 +483,8 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_trades_history', + MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 447f0052b..427e6c422 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -460,7 +460,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> def get_timerange(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame())) + mocker.patch('freqtrade.data.history.history_utils.load_pair_history', + MagicMock(return_value=pd.DataFrame())) mocker.patch('freqtrade.data.history.get_timerange', get_timerange) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) From 525550e4c75af8ae2c863e16a8fc0c47c367ae70 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 10:01:54 +0100 Subject: [PATCH 060/317] Fix typo in parameter transition --- freqtrade/configuration/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index aa453a392..99ca84f34 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -346,7 +346,7 @@ class Configuration: self._args_to_config(config, argname='dataformat_ohlcv', logstring='Using "{}" to store OHLCV data.') - self._args_to_config(config, argname='dataformat_trade', + self._args_to_config(config, argname='dataformat_trades', logstring='Using "{}" to store trades data.') def _process_runmode(self, config: Dict[str, Any]) -> None: From 28787a001ce03790bb4848ca55b09ebdf817aa17 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 10:27:49 +0100 Subject: [PATCH 061/317] Move convert functions to convert module --- freqtrade/configuration/arguments.py | 2 +- freqtrade/data/converter.py | 62 +++++++++++++++++++++++ freqtrade/data/history/jsondatahandler.py | 2 + freqtrade/utils.py | 58 +++------------------ tests/data/test_history.py | 3 +- 5 files changed, 73 insertions(+), 54 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 121bd23b4..2b9b362bf 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -47,7 +47,7 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] -ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to"] +ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 41a843e36..c6ec4fa2b 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -3,6 +3,7 @@ Functions to convert data from one format to another """ import logging from datetime import datetime, timezone +from typing import Any, Dict import pandas as pd from pandas import DataFrame, to_datetime @@ -175,3 +176,64 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame: # Drop 0 volume rows df_new = df_new.dropna() return df_new[DEFAULT_DATAFRAME_COLUMNS] + + +def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool): + """ + Convert trades from one format to another format. + :param config: Config dictionary + :param convert_from: Source format + :param convert_to: Target format + :param erase: Erase souce data (does not apply if source and target format are identical) + """ + from freqtrade.data.history.idatahandler import get_datahandler + src = get_datahandler(config['datadir'], convert_from) + trg = get_datahandler(config['datadir'], convert_to) + + if 'pairs' not in config: + config['pairs'] = src.trades_get_pairs(config['datadir']) + logger.info(f"Converting trades for {config['pairs']}") + + for pair in config['pairs']: + data = src.trades_load(pair=pair) + logger.info(f"Converting {len(data)} trades for {pair}") + trg.trades_store(pair, data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source Trade data for {pair}.") + src.trades_purge(pair=pair) + + +def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool): + """ + Convert ohlcv from one format to another format. + :param config: Config dictionary + :param convert_from: Source format + :param convert_to: Target format + :param erase: Erase souce data (does not apply if source and target format are identical) + """ + from freqtrade.data.history.idatahandler import get_datahandler + src = get_datahandler(config['datadir'], convert_from) + trg = get_datahandler(config['datadir'], convert_to) + timeframes = config.get('timeframes', [config.get('ticker_interval')]) + logger.info(f"Converting OHLCV for timeframe {timeframes}") + + if 'pairs' not in config: + config['pairs'] = [] + # Check timeframes or fall back to ticker_interval. + for timeframe in timeframes: + config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'], + timeframe)) + logger.info(f"Converting OHLCV for {config['pairs']}") + + for timeframe in timeframes: + for pair in config['pairs']: + data = src.ohlcv_load(pair=pair, timeframe=timeframe, + timerange=None, + fill_missing=False, + drop_incomplete=False, + startup_candles=0) + logger.info(f"Converting {len(data)} candles for {pair}") + trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source data for {pair} / {timeframe}") + src.ohlcv_purge(pair=pair, timeframe=timeframe) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index f40cf969f..14f643705 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -62,6 +62,8 @@ class JsonDataHandler(IDataHandler): :param pair: Pair to load data :param timeframe: Ticker timeframe (e.g. "5m") :param timerange: Limit data to be loaded to this timerange. + Optionally implemented by subclasses to avoid loading + all data where possible. :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename(self._datadir, pair, timeframe) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 1bb0f611a..b0ef7241b 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -17,8 +17,9 @@ from freqtrade.configuration import (Configuration, TimeRange, from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY +from freqtrade.data.converter import (convert_ohlcv_format, + convert_trades_format) from freqtrade.data.history import (convert_trades_to_ohlcv, - get_datahandlerclass, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) from freqtrade.exchange import (available_exchanges, ccxt_exchanges, @@ -248,55 +249,6 @@ def start_list_strategies(args: Dict[str, Any]) -> None: print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) -def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str): - """ - TODO: move this to converter.py (?) - """ - SrcClass = get_datahandlerclass(convert_from) - TrgClass = get_datahandlerclass(convert_to) - - if 'pairs' not in config: - config['pairs'] = SrcClass.trades_get_pairs(config['datadir']) - logger.info(f"Converting trades for {config['pairs']}") - src = SrcClass(config['datadir']) - trg = TrgClass(config['datadir']) - for pair in config['pairs']: - data = src.trades_load(pair=pair) - logger.info(f"Converting {len(data)} trades for {pair}") - trg.trades_store(pair, data) - - -def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str): - """ - TODO: move this to converter.py (?) - """ - SrcClass = get_datahandlerclass(convert_from) - TrgClass = get_datahandlerclass(convert_to) - timeframes = config.get('timeframes', [config.get('ticker_interval')]) - logger.info(f"Converting OHLCV for timeframe {timeframes}") - - if 'pairs' not in config: - config['pairs'] = [] - # Check timeframes or fall back to ticker_interval. - for timeframe in timeframes: - config['pairs'].extend(SrcClass.ohlcv_get_pairs(config['datadir'], - timeframe)) - logger.info(f"Converting OHLCV for {config['pairs']}") - - src = SrcClass(config['datadir']) - trg = TrgClass(config['datadir']) - - for timeframe in timeframes: - for pair in config['pairs']: - data = src.ohlcv_load(pair=pair, timeframe=timeframe, - timerange=None, - fill_missing=False, - drop_incomplete=False, - startup_candles=0) - logger.info(f"Converting {len(data)} candles for {pair}") - trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) - - def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ Convert data from one format to another @@ -304,10 +256,12 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if ohlcv: convert_ohlcv_format(config, - convert_from=args['format_from'], convert_to=args['format_to']) + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase']) else: convert_trades_format(config, - convert_from=args['format_from'], convert_to=args['format_to']) + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase']) def start_list_timeframes(args: Dict[str, Any]) -> None: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index f84e819ad..404f52e87 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -463,7 +463,8 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', + MagicMock()) ex = get_patched_exchange(mocker, default_conf) mocker.patch( From e7054adc496d54b060ea579d650e6d208f2be12a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 10:37:34 +0100 Subject: [PATCH 062/317] Add tests for start_convert_data --- tests/test_utils.py | 58 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 4cf7b5f23..e384a7633 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -6,13 +6,13 @@ import pytest from freqtrade import OperationalException from freqtrade.state import RunMode -from freqtrade.utils import (setup_utils_configuration, start_create_userdir, - start_download_data, start_hyperopt_list, - start_hyperopt_show, start_list_exchanges, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_hyperopt, - start_new_strategy, start_test_pairlist, - start_trading) +from freqtrade.utils import (setup_utils_configuration, start_convert_data, + start_create_userdir, start_download_data, + start_hyperopt_list, start_hyperopt_show, + start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, + start_new_hyperopt, start_new_strategy, + start_test_pairlist, start_trading) from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -821,3 +821,47 @@ def test_hyperopt_show(mocker, capsys, hyperopt_results): with pytest.raises(OperationalException, match="The index of the epoch to show should be less than 4."): start_hyperopt_show(pargs) + + +def test_convert_data(mocker, testdatadir): + ohlcv_mock = mocker.patch("freqtrade.utils.convert_ohlcv_format", MagicMock()) + trades_mock = mocker.patch("freqtrade.utils.convert_trades_format", MagicMock()) + args = [ + "convert-data", + "--format-from", + "json", + "--format-to", + "jsongz", + "--datadir", + str(testdatadir), + ] + pargs = get_args(args) + pargs['config'] = None + start_convert_data(pargs, True) + assert trades_mock.call_count == 0 + assert ohlcv_mock.call_count == 1 + assert ohlcv_mock.call_args[1]['convert_from'] == 'json' + assert ohlcv_mock.call_args[1]['convert_to'] == 'jsongz' + assert ohlcv_mock.call_args[1]['erase'] is False + + +def test_convert_data_trades(mocker, testdatadir): + ohlcv_mock = mocker.patch("freqtrade.utils.convert_ohlcv_format", MagicMock()) + trades_mock = mocker.patch("freqtrade.utils.convert_trades_format", MagicMock()) + args = [ + "convert-trade-data", + "--format-from", + "jsongz", + "--format-to", + "json", + "--datadir", + str(testdatadir), + ] + pargs = get_args(args) + pargs['config'] = None + start_convert_data(pargs, False) + assert ohlcv_mock.call_count == 0 + assert trades_mock.call_count == 1 + assert trades_mock.call_args[1]['convert_from'] == 'jsongz' + assert trades_mock.call_args[1]['convert_to'] == 'json' + assert trades_mock.call_args[1]['erase'] is False From 70f3ff046128491ca2310f967020308b92319c68 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 10:45:26 +0100 Subject: [PATCH 063/317] Add test for convert_trades_Format --- tests/data/test_converter.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index eb8a8e513..ac5472221 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -2,11 +2,13 @@ import logging from freqtrade.configuration.timerange import TimeRange -from freqtrade.data.converter import (ohlcv_fill_up_missing_data, +from freqtrade.data.converter import (convert_trades_format, + ohlcv_fill_up_missing_data, parse_ticker_dataframe, trim_dataframe) from freqtrade.data.history import (get_timerange, load_data, load_pair_history, validate_backtest_data) from tests.conftest import log_has +from tests.data.test_history import _backup_file, _clean_test_file def test_dataframe_correct_columns(result): @@ -188,3 +190,31 @@ def test_trim_dataframe(testdatadir) -> None: assert len(data_modify) == len(data) - 55 # first row matches 25th original row assert all(data_modify.iloc[0] == data.iloc[25]) + + +def test_convert_trades_format(mocker, default_conf, testdatadir): + file = testdatadir / "XRP_ETH-trades.json.gz" + file_new = testdatadir / "XRP_ETH-trades.json" + _backup_file(file, copy_file=True) + default_conf['datadir'] = testdatadir + + assert not file_new.exists() + + convert_trades_format(default_conf, convert_from='jsongz', + convert_to='json', erase=False) + + assert file_new.exists() + assert file.exists() + + # Remove original file + file.unlink() + # Convert back + convert_trades_format(default_conf, convert_from='json', + convert_to='jsongz', erase=True) + + assert file.exists() + assert not file_new.exists() + + _clean_test_file(file) + if file_new.exists(): + file_new.unlink() From 9e4fc00a0ffcc80dc5ddb91c2961f5cbc81d48a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 10:54:10 +0100 Subject: [PATCH 064/317] Add test for convert_ohlcv --- freqtrade/data/converter.py | 2 ++ tests/data/test_converter.py | 45 +++++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index c6ec4fa2b..abc754df0 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -135,6 +135,7 @@ def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: """ + TODO: This should get a dedicated test Gets order book list, returns dataframe with below format per suggested by creslin ------------------------------------------------------------------- b_sum b_size bids asks a_size a_sum @@ -160,6 +161,7 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame: """ Converts trades list to ohlcv list + TODO: This should get a dedicated test :param trades: List of trades, as returned by ccxt.fetch_trades. :param timeframe: Ticker timeframe to resample data to :return: ohlcv Dataframe. diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index ac5472221..a0ec2f46f 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -2,7 +2,8 @@ import logging from freqtrade.configuration.timerange import TimeRange -from freqtrade.data.converter import (convert_trades_format, +from freqtrade.data.converter import (convert_ohlcv_format, + convert_trades_format, ohlcv_fill_up_missing_data, parse_ticker_dataframe, trim_dataframe) from freqtrade.data.history import (get_timerange, load_data, @@ -218,3 +219,45 @@ def test_convert_trades_format(mocker, default_conf, testdatadir): _clean_test_file(file) if file_new.exists(): file_new.unlink() + + +def test_convert_ohlcv_format(mocker, default_conf, testdatadir): + file1 = testdatadir / "XRP_ETH-5m.json" + file1_new = testdatadir / "XRP_ETH-5m.json.gz" + file2 = testdatadir / "XRP_ETH-1m.json" + file2_new = testdatadir / "XRP_ETH-1m.json.gz" + _backup_file(file1, copy_file=True) + _backup_file(file2, copy_file=True) + default_conf['datadir'] = testdatadir + default_conf['pairs'] = ['XRP_ETH'] + default_conf['timeframes'] = ['1m', '5m'] + + assert not file1_new.exists() + assert not file2_new.exists() + + convert_ohlcv_format(default_conf, convert_from='json', + convert_to='jsongz', erase=False) + + assert file1_new.exists() + assert file2_new.exists() + assert file1.exists() + assert file2.exists() + + # Remove original files + file1.unlink() + file2.unlink() + # Convert back + convert_ohlcv_format(default_conf, convert_from='jsongz', + convert_to='json', erase=True) + + assert file1.exists() + assert file2.exists() + assert not file1_new.exists() + assert not file2_new.exists() + + _clean_test_file(file1) + _clean_test_file(file2) + if file1_new.exists(): + file1_new.unlink() + if file2_new.exists(): + file2_new.unlink() From 66d18575a741057ac9a763b4b72a3e8cd91e7883 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 11:10:31 +0100 Subject: [PATCH 065/317] Implement abstract interface --- freqtrade/data/history/idatahandler.py | 97 ++++++++++++++++++++++- freqtrade/data/history/jsondatahandler.py | 1 - 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index cee43dcef..20022fc38 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -24,7 +24,102 @@ class IDataHandler(ABC): def __init__(self, datadir: Path) -> None: self._datadir = datadir - # TODO: create abstract interface + @abstractclassmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs with ohlcv data available in this datadir + for the specified timeframe + :param datadir: Directory to search for ohlcv files + :param timeframe: Timeframe to search pairs for + :return: List of Pairs + """ + + @abstractmethod + def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Store data in json format "values". + format looks as follows: + [[,,,,]] + :param pair: Pair - used to generate filename + :timeframe: Timeframe - used to generate filename + :data: Dataframe containing OHLCV data + :return: None + """ + + @abstractmethod + def _ohlcv_load(self, pair: str, timeframe: str, + timerange: Optional[TimeRange] = None, + ) -> DataFrame: + """ + Internal method used to load data for one pair from disk. + Implements the loading and conversation to a Pandas dataframe. + Timerange trimming and dataframe validation happens outside of this method. + :param pair: Pair to load data + :param timeframe: Ticker timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange. + Optionally implemented by subclasses to avoid loading + all data where possible. + :return: DataFrame with ohlcv data, or empty DataFrame + """ + + @abstractmethod + def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :param timeframe: Ticker timeframe (e.g. "5m") + :return: True when deleted, false if file did not exist. + """ + + @abstractmethod + def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Append data to existing data structures + :param pair: Pair + :param timeframe: Timeframe this ohlcv data is for + :param data: Data to append. + """ + + @abstractclassmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs for which trade data is available in this + :param datadir: Directory to search for ohlcv files + :return: List of Pairs + """ + + @abstractmethod + def trades_store(self, pair: str, data: List[Dict]) -> None: + """ + Store trades data (list of Dicts) to file + :param pair: Pair - used for filename + :param data: List of Dicts containing trade data + """ + + @abstractmethod + def trades_append(self, pair: str, data: List[Dict]): + """ + Append data to existing files + :param pair: Pair - used for filename + :param data: List of Dicts containing trade data + """ + + @abstractmethod + def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]: + """ + Load a pair from file, either .json.gz or .json + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + + @abstractmethod + def trades_purge(self, pair: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :return: True when deleted, false if file did not exist. + """ def ohlcv_load(self, pair, timeframe: str, timerange: Optional[TimeRange] = None, diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 14f643705..dcfc249aa 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -96,7 +96,6 @@ class JsonDataHandler(IDataHandler): :param pair: Pair :param timeframe: Timeframe this ohlcv data is for :param data: Data to append. - """ raise NotImplementedError() From e2a00c03d626fb6242968c19402ccc5ebf7e505b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 11:24:37 +0100 Subject: [PATCH 066/317] Document convert options --- config_full.json.example | 4 +- docs/data-download.md | 146 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) diff --git a/config_full.json.example b/config_full.json.example index b9631f63d..108fdd534 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -127,5 +127,7 @@ "heartbeat_interval": 60 }, "strategy": "DefaultStrategy", - "strategy_path": "user_data/strategies/" + "strategy_path": "user_data/strategies/", + "dataformat_ohlcv": "json", + "dataformat_trades": "jsongz" } diff --git a/docs/data-download.md b/docs/data-download.md index 1f03b124a..c5fb744ab 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -12,6 +12,152 @@ Otherwise `--exchange` becomes mandatory. If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. +### Usage + +``` +usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] + [--pairs-file FILE] [--days INT] [--dl-trades] [--exchange EXCHANGE] + [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]] + [--erase] [--data-format {json,jsongz}] [--data-format-trades {json,jsongz}] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space-separated. + --pairs-file FILE File containing a list of pairs to download. + --days INT Download data for given number of days. + --dl-trades Download trades instead of OHLCV data. The bot will resample trades to the desired timeframe as specified as + --timeframes/-t. + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. + -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...] + Specify which tickers to download. Space-separated list. Default: `1m 5m`. + --erase Clean all existing data for the selected exchange/pairs/timeframes. + --data-format {json,jsongz} + Storage format for downloaded ohlcv data. (default: `json`). + --data-format-trades {json,jsongz} + Storage format for downloaded trades data. (default: `jsongz`). + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` + to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` + +### Data format + +Freqtrade currently supports 2 dataformats, `json` and `jsongz`, a zipped version of json files. +By default, OHLCV data is stored as json data, while trades data is stored as `jsongz` data. + +This can be changed via the `--data-format` and `--data-format-trades` parameters respectivly. + +If the default dataformat has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well. + +!!! Note + You can convert between data-formats using the [convert-data](#subcommand-convert-data) and [convert-trade-data](#subcommand-convert-trade-data) methods. + +#### Subcommand convert data + +``` +usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [-p PAIRS [PAIRS ...]] --format-from + {json,jsongz} --format-to {json,jsongz} + [--erase] + [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space- + separated. + --format-from {json,jsongz} + Source format for data conversation. + --format-to {json,jsongz} + Destination format for data conversation. + --erase Clean all existing data for the selected + exchange/pairs/timeframes. + -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...] + Specify which tickers to download. Space-separated + list. Default: `1m 5m`. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` + +##### Example converting data + +The following command will convert all Candle data available in `~/.freqtrade/data/binance` from json to jsongz, saving diskspace in the process. +It'll also remove source files (`--erase` parameter). + +``` bash +freqtrade convert-data --format-from json --format-to jsongz --data-dir ~/.freqtrade/data/binance -t 5m 15m --erase +``` + +#### Subcommand convert-trade data + +``` +usage: freqtrade convert-trade-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [-p PAIRS [PAIRS ...]] --format-from + {json,jsongz} --format-to {json,jsongz} + [--erase] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space- + separated. + --format-from {json,jsongz} + Source format for data conversation. + --format-to {json,jsongz} + Destination format for data conversation. + --erase Clean all existing data for the selected + exchange/pairs/timeframes. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` + +##### Example converting trades + +The following command will convert all available trade-data in `~/.freqtrade/data/kraken` from json to jsongz, saving diskspace in the process. +It'll also remove source files (`--erase` parameter). + +``` bash +freqtrade convert-trade-data --format-from jsongz --format-to json --data-dir ~/.freqtrade/data/kraken --erase +``` + ### Pairs file In alternative to the whitelist from `config.json`, a `pairs.json` file can be used. From ae1b28aab756adbef0fc64b0194a3b38a2ad0686 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 14:32:11 +0100 Subject: [PATCH 067/317] Remove get_datahandlerclass from package exposes --- freqtrade/data/history/__init__.py | 2 +- tests/data/test_history.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/history/__init__.py b/freqtrade/data/history/__init__.py index 572c063fc..23f635a98 100644 --- a/freqtrade/data/history/__init__.py +++ b/freqtrade/data/history/__init__.py @@ -11,4 +11,4 @@ from .history_utils import (convert_trades_to_ohlcv, # noqa: F401 refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data, validate_backtest_data) -from .idatahandler import get_datahandler, get_datahandlerclass # noqa: F401 +from .idatahandler import get_datahandler # noqa: F401 diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 404f52e87..2341673db 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -13,13 +13,13 @@ from pandas.testing import assert_frame_equal from freqtrade.configuration import TimeRange from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.data.history import get_datahandler, get_datahandlerclass from freqtrade.data.history.history_utils import ( _download_pair_history, _download_trades_history, _load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange, load_data, load_pair_history, refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data, validate_backtest_data) -from freqtrade.data.history.idatahandler import IDataHandler +from freqtrade.data.history.idatahandler import (IDataHandler, get_datahandler, + get_datahandlerclass) from freqtrade.data.history.jsondatahandler import (JsonDataHandler, JsonGzDataHandler) from freqtrade.exchange import timeframe_to_minutes From 6b5983339d8d94d90c51f0dbb51f1948fed4bbea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 14:47:30 +0100 Subject: [PATCH 068/317] Require dataformat entries in configuration --- freqtrade/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 95ec71552..a7f569f63 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -294,5 +294,7 @@ CONF_SCHEMA = { 'stoploss', 'minimal_roi', 'internals', + 'dataformat_ohlcv', + 'dataformat_trades', ] } From f4a532ef6d3ed63751de5c688338af1036741d7f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 14:57:39 +0100 Subject: [PATCH 069/317] Pass format to load_data --- freqtrade/data/history/history_utils.py | 2 +- freqtrade/edge/__init__.py | 1 + freqtrade/optimize/backtesting.py | 1 + freqtrade/plot/plotting.py | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 42f1c9be7..1f0459379 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -72,7 +72,7 @@ def load_data(datadir: Path, :param fill_up_missing: Fill missing values with "No action"-candles :param startup_candles: Additional candles to load at the start of the period :param fail_without_data: Raise OperationalException if no data is found. - :param data_handler: Initialized data-handler to use. + :param data_format: Data format which should be used. Defaults to json :return: dict(:) """ result: Dict[str, DataFrame] = {} diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 9ad2485ef..44eaad717 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -108,6 +108,7 @@ class Edge: timeframe=self.strategy.ticker_interval, timerange=self._timerange, startup_candles=self.strategy.startup_candle_count, + data_format=self.config.get('dataformat_ohlcv', 'json'), ) if not data: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 98ee71a60..e49c0bd80 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -116,6 +116,7 @@ class Backtesting: timerange=timerange, startup_candles=self.required_startup, fail_without_data=True, + data_format=self.config.get('dataformat_ohlcv', 'json'), ) min_date, max_date = history.get_timerange(data) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 0ef71ed82..7c7f10fdc 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -42,6 +42,7 @@ def init_plotscript(config): pairs=pairs, timeframe=config.get('ticker_interval', '5m'), timerange=timerange, + data_format=config.get('dataformat_ohlcv', 'json'), ) trades = load_trades(config['trade_source'], From 814cc20c6b94ea0c34b8303dbb14c373f6f0290f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Dec 2019 19:58:41 +0100 Subject: [PATCH 070/317] Remove potential circular import --- freqtrade/data/converter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index abc754df0..49a2a25bc 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -8,7 +8,6 @@ from typing import Any, Dict import pandas as pd from pandas import DataFrame, to_datetime -from freqtrade.configuration.timerange import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS logger = logging.getLogger(__name__) @@ -116,7 +115,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) return df -def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date') -> DataFrame: +def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date') -> DataFrame: """ Trim dataframe based on given timerange :param df: Dataframe to trim From f82c4346b6ce65c5ded54a63b3931780cd349a98 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Jan 2020 09:55:02 +0100 Subject: [PATCH 071/317] data conversion, not data conversation * we're not talking to the data yet ... --- docs/data-download.md | 8 ++++---- freqtrade/configuration/arguments.py | 8 ++++---- freqtrade/configuration/cli_options.py | 4 ++-- freqtrade/data/history/history_utils.py | 2 +- freqtrade/data/history/idatahandler.py | 2 +- freqtrade/data/history/jsondatahandler.py | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index c5fb744ab..f747123d0 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -78,9 +78,9 @@ optional arguments: Show profits for only these pairs. Pairs are space- separated. --format-from {json,jsongz} - Source format for data conversation. + Source format for data conversion. --format-to {json,jsongz} - Destination format for data conversation. + Destination format for data conversion. --erase Clean all existing data for the selected exchange/pairs/timeframes. -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...] @@ -127,9 +127,9 @@ optional arguments: Show profits for only these pairs. Pairs are space- separated. --format-from {json,jsongz} - Source format for data conversation. + Source format for data conversion. --format-to {json,jsongz} - Destination format for data conversation. + Destination format for data conversion. --erase Clean all existing data for the selected exchange/pairs/timeframes. diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 2b9b362bf..718b6dedc 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -264,14 +264,14 @@ class Arguments: convert_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=True)) self._build_args(optionlist=ARGS_CONVERT_DATA_OHLCV, parser=convert_data_cmd) - # Add convert-data subcommand - convert_data_cmd = subparsers.add_parser( + # Add convert-trade-data subcommand + convert_trade_data_cmd = subparsers.add_parser( 'convert-trade-data', help='Convert trade-data from one format to another.', parents=[_common_parser], ) - convert_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False)) - self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_data_cmd) + convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False)) + self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd) # Add Plotting subcommand plot_dataframe_cmd = subparsers.add_parser( diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index c80bea393..0289d3fd0 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -334,13 +334,13 @@ AVAILABLE_CLI_OPTIONS = { ), "format_from": Arg( '--format-from', - help='Source format for data conversation.', + help='Source format for data conversion.', choices=constants.AVAILABLE_DATAHANDLERS, required=True, ), "format_to": Arg( '--format-to', - help='Destination format for data conversation.', + help='Destination format for data conversion.', choices=constants.AVAILABLE_DATAHANDLERS, required=True, ), diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 2d11b229b..c0578a32b 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -132,7 +132,7 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona start = None if timerange: if timerange.starttype == 'date': - # TODO: convert to date for conversation + # TODO: convert to date for conversion start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) # Intentionally don't pass timerange in - since we need to load the full dataset. diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 20022fc38..df03e7713 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -52,7 +52,7 @@ class IDataHandler(ABC): ) -> DataFrame: """ Internal method used to load data for one pair from disk. - Implements the loading and conversation to a Pandas dataframe. + Implements the loading and conversion to a Pandas dataframe. Timerange trimming and dataframe validation happens outside of this method. :param pair: Pair to load data :param timeframe: Ticker timeframe (e.g. "5m") diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index dcfc249aa..7f0643862 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -57,7 +57,7 @@ class JsonDataHandler(IDataHandler): ) -> DataFrame: """ Internal method used to load data for one pair from disk. - Implements the loading and conversation to a Pandas dataframe. + Implements the loading and conversion to a Pandas dataframe. Timerange trimming and dataframe validation happens outside of this method. :param pair: Pair to load data :param timeframe: Ticker timeframe (e.g. "5m") From bc6a10353bf4c704003a319fd953026896c33533 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 4 Jan 2020 05:07:51 +0300 Subject: [PATCH 072/317] Introduce pair_to_filename() --- freqtrade/misc.py | 6 ++++++ freqtrade/plot/plotting.py | 12 +++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index ed37ace3a..e6ebc8d65 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -92,6 +92,12 @@ def file_load_json(file): return pairdata +def pair_to_filename(pair: str) -> str: + for ch in ['/', ' ', '.']: + pair = pair.replace(ch, '_') + return pair + + def format_ms_time(date: int) -> str: """ convert MS date to readable format. diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 6b2d426e7..7c9c9f985 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -3,12 +3,14 @@ from pathlib import Path from typing import Any, Dict, List import pandas as pd + from freqtrade.configuration import TimeRange -from freqtrade.data import history -from freqtrade.data.converter import trim_dataframe from freqtrade.data.btanalysis import (combine_tickers_with_mean, create_cum_profit, extract_trades_of_period, load_trades) +from freqtrade.data.converter import trim_dataframe +from freqtrade.data.history import load_data +from freqtrade.misc import pair_to_filename from freqtrade.resolvers import StrategyResolver logger = logging.getLogger(__name__) @@ -37,7 +39,7 @@ def init_plotscript(config): # Set timerange to use timerange = TimeRange.parse_timerange(config.get("timerange")) - tickers = history.load_data( + tickers = load_data( datadir=config.get("datadir"), pairs=pairs, timeframe=config.get('ticker_interval', '5m'), @@ -306,8 +308,8 @@ def generate_plot_filename(pair, timeframe) -> str: """ Generate filenames per pair/timeframe to be used for storing plots """ - pair_name = pair.replace("/", "_") - file_name = 'freqtrade-plot-' + pair_name + '-' + timeframe + '.html' + pair_s = pair_to_filename(pair) + file_name = 'freqtrade-plot-' + pair_s + '-' + timeframe + '.html' logger.info('Generate plot file for %s', pair) From 4eaaec9d1a0bee87a6ce2b60b2044b5bd52164b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Jan 2020 10:36:08 +0100 Subject: [PATCH 073/317] Implement pair_to_filename to datahandler includes tests - taken from #2744 and modified to adapt to new structure --- freqtrade/data/history/jsondatahandler.py | 8 ++--- freqtrade/misc.py | 2 +- tests/data/test_history.py | 36 ++++++++++++++++------- tests/test_misc.py | 25 +++++++++++++++- 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 7f0643862..7da1477af 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -30,7 +30,7 @@ class JsonDataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name) for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] # Check if regex found something and only return these results - return [match[0].replace('_', '/') for match in _tmp if match] + return [misc.pair_to_filename(match[0]) for match in _tmp if match] def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: """ @@ -109,7 +109,7 @@ class JsonDataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name) for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] # Check if regex found something and only return these results to avoid exceptions. - return [match[0].replace('_', '/') for match in _tmp if match] + return [misc.pair_to_filename(match[0]) for match in _tmp if match] def trades_store(self, pair: str, data: List[Dict]) -> None: """ @@ -157,7 +157,7 @@ class JsonDataHandler(IDataHandler): @classmethod def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: - pair_s = pair.replace("/", "_") + pair_s = misc.pair_to_filename(pair) filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') return filename @@ -167,7 +167,7 @@ class JsonDataHandler(IDataHandler): @classmethod def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: - pair_s = pair.replace("/", "_") + pair_s = misc.pair_to_filename(pair) filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') return filename diff --git a/freqtrade/misc.py b/freqtrade/misc.py index e6ebc8d65..f012400c4 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -93,7 +93,7 @@ def file_load_json(file): def pair_to_filename(pair: str) -> str: - for ch in ['/', ' ', '.']: + for ch in ['/', '-', ' ', '.', '@', '$', '+', ':']: pair = pair.replace(ch, '_') return pair diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 2341673db..39000e508 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -144,23 +144,39 @@ def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) -def test_json_pair_data_filename(): - fn = JsonDataHandler._pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m') +@pytest.mark.parametrize("pair,expected_result", [ + ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json'), + ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json'), + ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json'), + (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json'), + ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json'), + ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json'), +]) +def test_json_pair_data_filename(pair, expected_result): + fn = JsonDataHandler._pair_data_filename(Path('freqtrade/hello/world'), pair, '5m') assert isinstance(fn, Path) - assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json') - fn = JsonGzDataHandler._pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m') + assert fn == Path(expected_result) + fn = JsonGzDataHandler._pair_data_filename(Path('freqtrade/hello/world'), pair, '5m') assert isinstance(fn, Path) - assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json.gz') + assert fn == Path(expected_result + '.gz') -def test_json_pair_trades_filename(): - fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC') +@pytest.mark.parametrize("pair,expected_result", [ + ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'), + ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), + ("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'), + (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'), + ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-trades.json'), + ("ACC_OLD_BTC", 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'), +]) +def test_json_pair_trades_filename(pair, expected_result): + fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) assert isinstance(fn, Path) - assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json') + assert fn == Path(expected_result) - fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC') + fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) assert isinstance(fn, Path) - assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json.gz') + assert fn == Path(expected_result + '.gz') def test_load_cached_data_for_updating(mocker, testdatadir) -> None: diff --git a/tests/test_misc.py b/tests/test_misc.py index c5bf06311..83e008466 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -4,9 +4,12 @@ import datetime from pathlib import Path from unittest.mock import MagicMock +import pytest + from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json, - file_load_json, format_ms_time, plural, shorten_date) + file_load_json, format_ms_time, pair_to_filename, + plural, shorten_date) def test_shorten_date() -> None: @@ -57,6 +60,26 @@ def test_file_load_json(mocker, testdatadir) -> None: assert ret +@pytest.mark.parametrize("pair,expected_result", [ + ("ETH/BTC", 'ETH_BTC'), + ("Fabric Token/ETH", 'Fabric_Token_ETH'), + ("ETHH20", 'ETHH20'), + (".XBTBON2H", '_XBTBON2H'), + ("ETHUSD.d", 'ETHUSD_d'), + ("ADA-0327", 'ADA_0327'), + ("BTC-USD-200110", 'BTC_USD_200110'), + ("F-AKRO/USDT", 'F_AKRO_USDT'), + ("LC+/ETH", 'LC__ETH'), + ("CMT@18/ETH", 'CMT_18_ETH'), + ("LBTC:1022/SAI", 'LBTC_1022_SAI'), + ("$PAC/BTC", '_PAC_BTC'), + ("ACC_OLD/BTC", 'ACC_OLD_BTC'), +]) +def test_pair_to_filename(pair, expected_result): + pair_s = pair_to_filename(pair) + assert pair_s == expected_result + + def test_format_ms_time() -> None: # Date 2018-04-10 18:02:01 date_in_epoch_ms = 1523383321000 From 41945138ac569947ed5ae788aeb7d5486bfc2c2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Jan 2020 13:35:36 +0100 Subject: [PATCH 074/317] Converting pairs from filename to pair corrected --- freqtrade/data/history/jsondatahandler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 7da1477af..17b9fd7d7 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -30,7 +30,7 @@ class JsonDataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name) for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] # Check if regex found something and only return these results - return [misc.pair_to_filename(match[0]) for match in _tmp if match] + return [match[0].replace('_', '/') for match in _tmp if match] def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: """ @@ -109,7 +109,7 @@ class JsonDataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name) for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] # Check if regex found something and only return these results to avoid exceptions. - return [misc.pair_to_filename(match[0]) for match in _tmp if match] + return [match[0].replace('_', '/') for match in _tmp if match] def trades_store(self, pair: str, data: List[Dict]) -> None: """ From 8d2e0bfd628c13ff51e5962107b7c0fed5db78b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jan 2020 13:12:28 +0100 Subject: [PATCH 075/317] Move rate-calcuation for stoploss-limit order to exchange --- freqtrade/exchange/binance.py | 8 ++++++-- freqtrade/exchange/exchange.py | 3 ++- freqtrade/freqtradebot.py | 5 +---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 12326f083..15796bdcb 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -32,13 +32,17 @@ class Binance(Exchange): return super().get_order_book(pair, limit) - def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: + def stoploss_limit(self, pair: str, amount: float, stop_price: float, + order_types: Dict) -> Dict: """ creates a stoploss limit order. this stoploss-limit is binance-specific. It may work with a limited number of other exchanges, but this has not been tested yet. - """ + # Limit price threshold: As limit price should always be below stop-price + LIMIT_PRICE_PCT = order_types.get('stoploss_on_exchange_limit_ratio', 0.99) + rate = stop_price * LIMIT_PRICE_PCT + ordertype = "stop_loss_limit" stop_price = self.price_to_precision(pair, stop_price) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 87c189457..4c5ef823b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -519,7 +519,8 @@ class Exchange: return self.create_order(pair, ordertype, 'sell', amount, rate, params) - def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: + def stoploss_limit(self, pair: str, amount: float, stop_price: float, + order_types: Dict) -> Dict: """ creates a stoploss limit order. Since ccxt does not unify stoploss-limit orders yet, this needs to be implemented in each diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e712892f1..1a3097c25 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -636,13 +636,10 @@ class FreqtradeBot: Force-sells the pair (using EmergencySell reason) in case of Problems creating the order. :return: True if the order succeeded, and False in case of problems. """ - # Limit price threshold: As limit price should always be below stop-price - LIMIT_PRICE_PCT = self.strategy.order_types.get('stoploss_on_exchange_limit_ratio', 0.99) - try: stoploss_order = self.exchange.stoploss_limit(pair=trade.pair, amount=trade.amount, stop_price=stop_price, - rate=rate * LIMIT_PRICE_PCT) + order_types=self.strategy.order_types) trade.stoploss_order_id = str(stoploss_order['id']) return True except InvalidOrderException as e: From da0af489a2cd8cc437752e308cab9e44ffc38c7a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jan 2020 13:25:41 +0100 Subject: [PATCH 076/317] Adjust tests to pass in order_types instead of rate --- tests/exchange/test_binance.py | 19 ++++++++++--------- tests/exchange/test_exchange.py | 2 +- tests/test_freqtradebot.py | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 4bc918c3d..bda4946b4 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -28,11 +28,12 @@ def test_stoploss_limit_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, + order_types={'stoploss_on_exchange_limit_ratio': 1.05}) api_mock.create_order.reset_mock() - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) assert 'id' in order assert 'info' in order @@ -41,30 +42,29 @@ def test_stoploss_limit_order(default_conf, mocker): assert api_mock.create_order.call_args[0][1] == order_type assert api_mock.create_order.call_args[0][2] == 'sell' assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][4] == 200 assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} # test exception handling with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) with pytest.raises(InvalidOrderException): api_mock.create_order = MagicMock( side_effect=ccxt.InvalidOrder("binance Order would trigger immediately.")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) with pytest.raises(OperationalException, match=r".*DeadBeef.*"): api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) def test_stoploss_limit_order_dry_run(default_conf, mocker): @@ -77,11 +77,12 @@ def test_stoploss_limit_order_dry_run(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, + order_types={'stoploss_on_exchange_limit_ratio': 1.05}) api_mock.create_order.reset_mock() - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) assert 'id' in order assert 'info' in order diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 7064d76e1..be40f2192 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1761,7 +1761,7 @@ def test_get_fee(default_conf, mocker, exchange_name): def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, 'bittrex') with pytest.raises(OperationalException, match=r"stoploss_limit is not implemented .*"): - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) def test_merge_ft_has_dict(default_conf, mocker): diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5a4820f2f..2aa1548f8 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1315,7 +1315,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') stoploss_order_mock.assert_called_once_with(amount=85.25149190110828, pair='ETH/BTC', - rate=0.00002344 * 0.95 * 0.99, + order_types=freqtrade.strategy.order_types, stop_price=0.00002344 * 0.95) @@ -1492,7 +1492,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, cancel_order_mock.assert_called_once_with(100, 'NEO/BTC') stoploss_order_mock.assert_called_once_with(amount=2131074.168797954, pair='NEO/BTC', - rate=0.00002344 * 0.99 * 0.99, + order_types=freqtrade.strategy.order_types, stop_price=0.00002344 * 0.99) From 256fc2e78cc532714b6a8f91d0783b809349bb0c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jan 2020 13:30:56 +0100 Subject: [PATCH 077/317] Rename stoploss_limit to stoploss --- freqtrade/exchange/binance.py | 3 +-- freqtrade/exchange/exchange.py | 6 +++--- freqtrade/freqtradebot.py | 6 +++--- tests/exchange/test_binance.py | 20 ++++++++++---------- tests/exchange/test_exchange.py | 2 +- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 15796bdcb..d08726cf0 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -32,8 +32,7 @@ class Binance(Exchange): return super().get_order_book(pair, limit) - def stoploss_limit(self, pair: str, amount: float, stop_price: float, - order_types: Dict) -> Dict: + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: """ creates a stoploss limit order. this stoploss-limit is binance-specific. diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 4c5ef823b..121a8c636 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -519,10 +519,10 @@ class Exchange: return self.create_order(pair, ordertype, 'sell', amount, rate, params) - def stoploss_limit(self, pair: str, amount: float, stop_price: float, - order_types: Dict) -> Dict: + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: """ - creates a stoploss limit order. + creates a stoploss order. + The precise ordertype is determined by the order_types dict or exchange default. Since ccxt does not unify stoploss-limit orders yet, this needs to be implemented in each exchange's subclass. The exception below should never raise, since we disallow diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1a3097c25..a4b0ab806 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -637,9 +637,9 @@ class FreqtradeBot: :return: True if the order succeeded, and False in case of problems. """ try: - stoploss_order = self.exchange.stoploss_limit(pair=trade.pair, amount=trade.amount, - stop_price=stop_price, - order_types=self.strategy.order_types) + stoploss_order = self.exchange.stoploss(pair=trade.pair, amount=trade.amount, + stop_price=stop_price, + order_types=self.strategy.order_types) trade.stoploss_order_id = str(stoploss_order['id']) return True except InvalidOrderException as e: diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index bda4946b4..fdf3d7435 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -28,12 +28,12 @@ def test_stoploss_limit_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, - order_types={'stoploss_on_exchange_limit_ratio': 1.05}) + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=190, + order_types={'stoploss_on_exchange_limit_ratio': 1.05}) api_mock.create_order.reset_mock() - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) assert 'id' in order assert 'info' in order @@ -48,23 +48,23 @@ def test_stoploss_limit_order(default_conf, mocker): with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) with pytest.raises(InvalidOrderException): api_mock.create_order = MagicMock( side_effect=ccxt.InvalidOrder("binance Order would trigger immediately.")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) with pytest.raises(OperationalException, match=r".*DeadBeef.*"): api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) def test_stoploss_limit_order_dry_run(default_conf, mocker): @@ -77,12 +77,12 @@ def test_stoploss_limit_order_dry_run(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, - order_types={'stoploss_on_exchange_limit_ratio': 1.05}) + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=190, + order_types={'stoploss_on_exchange_limit_ratio': 1.05}) api_mock.create_order.reset_mock() - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) assert 'id' in order assert 'info' in order diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index be40f2192..7c0c72491 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1761,7 +1761,7 @@ def test_get_fee(default_conf, mocker, exchange_name): def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, 'bittrex') with pytest.raises(OperationalException, match=r"stoploss_limit is not implemented .*"): - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) def test_merge_ft_has_dict(default_conf, mocker): From 16b34e11cad56216d7a8afde5a4ad73e98cc513b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jan 2020 14:39:51 +0100 Subject: [PATCH 078/317] Complete rename of stoploss_limit to stoploss --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 4 +-- tests/test_freqtradebot.py | 60 ++++++++++++++++----------------- tests/test_integration.py | 4 +-- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 121a8c636..bef92750c 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -530,7 +530,7 @@ class Exchange: Note: Changes to this interface need to be applied to all sub-classes too. """ - raise OperationalException(f"stoploss_limit is not implemented for {self.name}.") + raise OperationalException(f"stoploss is not implemented for {self.name}.") @retrier def get_balance(self, currency: str) -> float: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 7c0c72491..680e69764 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1758,9 +1758,9 @@ def test_get_fee(default_conf, mocker, exchange_name): 'get_fee', 'calculate_fee', symbol="ETH/BTC") -def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker): +def test_stoploss_order_unsupported_exchange(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, 'bittrex') - with pytest.raises(OperationalException, match=r"stoploss_limit is not implemented .*"): + with pytest.raises(OperationalException, match=r"stoploss is not implemented .*"): exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 2aa1548f8..a33d47f34 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1031,8 +1031,8 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=limit_buy_order['amount']) - stoploss_limit = MagicMock(return_value={'id': 13434334}) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + stoploss = MagicMock(return_value={'id': 13434334}) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) freqtrade = FreqtradeBot(default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -1045,13 +1045,13 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None freqtrade.exit_positions(trades) assert trade.stoploss_order_id == '13434334' - assert stoploss_limit.call_count == 1 + assert stoploss.call_count == 1 assert trade.is_open is True def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, limit_buy_order, limit_sell_order) -> None: - stoploss_limit = MagicMock(return_value={'id': 13434334}) + stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -1064,7 +1064,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss_limit=stoploss_limit + stoploss=stoploss ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -1078,7 +1078,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, trade.stoploss_order_id = None assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert stoploss_limit.call_count == 1 + assert stoploss.call_count == 1 assert trade.stoploss_order_id == "13434334" # Second case: when stoploss is set but it is not yet hit @@ -1102,10 +1102,10 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, canceled_stoploss_order = MagicMock(return_value={'status': 'canceled'}) mocker.patch('freqtrade.exchange.Exchange.get_order', canceled_stoploss_order) - stoploss_limit.reset_mock() + stoploss.reset_mock() assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert stoploss_limit.call_count == 1 + assert stoploss.call_count == 1 assert trade.stoploss_order_id == "13434334" # Fourth case: when stoploss is set and it is hit @@ -1132,7 +1132,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, assert trade.is_open is False mocker.patch( - 'freqtrade.exchange.Exchange.stoploss_limit', + 'freqtrade.exchange.Exchange.stoploss', side_effect=DependencyException() ) freqtrade.handle_stoploss_on_exchange(trade) @@ -1142,11 +1142,11 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, # Fifth case: get_order returns InvalidOrder # It should try to add stoploss order trade.stoploss_order_id = 100 - stoploss_limit.reset_mock() + stoploss.reset_mock() mocker.patch('freqtrade.exchange.Exchange.get_order', side_effect=InvalidOrderException()) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) freqtrade.handle_stoploss_on_exchange(trade) - assert stoploss_limit.call_count == 1 + assert stoploss.call_count == 1 def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, @@ -1165,7 +1165,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, get_order=MagicMock(return_value={'status': 'canceled'}), - stoploss_limit=MagicMock(side_effect=DependencyException()), + stoploss=MagicMock(side_effect=DependencyException()), ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -1199,7 +1199,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, sell=sell_mock, get_fee=fee, get_order=MagicMock(return_value={'status': 'canceled'}), - stoploss_limit=MagicMock(side_effect=InvalidOrderException()), + stoploss=MagicMock(side_effect=InvalidOrderException()), ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -1229,7 +1229,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set - stoploss_limit = MagicMock(return_value={'id': 13434334}) + stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -1241,7 +1241,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss_limit=stoploss_limit + stoploss=stoploss ) # enabling TSL @@ -1296,7 +1296,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock() mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order_mock) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_order_mock) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss_order_mock) # stoploss should not be updated as the interval is 60 seconds assert freqtrade.handle_trade(trade) is False @@ -1322,7 +1322,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, caplog, limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set - stoploss_limit = MagicMock(return_value={'id': 13434334}) + stoploss = MagicMock(return_value={'id': 13434334}) patch_exchange(mocker) mocker.patch.multiple( @@ -1335,7 +1335,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss_limit=stoploss_limit + stoploss=stoploss ) # enabling TSL @@ -1375,12 +1375,12 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c assert log_has_re(r"Could not cancel stoploss order abcd for pair ETH/BTC.*", caplog) # Still try to create order - assert stoploss_limit.call_count == 1 + assert stoploss.call_count == 1 # Fail creating stoploss order caplog.clear() cancel_mock = mocker.patch("freqtrade.exchange.Exchange.cancel_order", MagicMock()) - mocker.patch("freqtrade.exchange.Exchange.stoploss_limit", side_effect=DependencyException()) + mocker.patch("freqtrade.exchange.Exchange.stoploss", side_effect=DependencyException()) freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) assert cancel_mock.call_count == 1 assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog) @@ -1390,7 +1390,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set - stoploss_limit = MagicMock(return_value={'id': 13434334}) + stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) @@ -1406,7 +1406,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss_limit=stoploss_limit + stoploss=stoploss ) # enabling TSL @@ -1459,7 +1459,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock() mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order_mock) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_order_mock) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss_order_mock) # price goes down 5% mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ @@ -2423,7 +2423,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke default_conf['exchange']['name'] = 'binance' rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) - stoploss_limit = MagicMock(return_value={ + stoploss = MagicMock(return_value={ 'id': 123, 'info': { 'foo': 'bar' @@ -2437,7 +2437,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke get_fee=fee, amount_to_precision=lambda s, x, y: y, price_to_precision=lambda s, x, y: y, - stoploss_limit=stoploss_limit, + stoploss=stoploss, cancel_order=cancel_order, ) @@ -2482,14 +2482,14 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f price_to_precision=lambda s, x, y: y, ) - stoploss_limit = MagicMock(return_value={ + stoploss = MagicMock(return_value={ 'id': 123, 'info': { 'foo': 'bar' } }) - mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) freqtrade = FreqtradeBot(default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -2507,7 +2507,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f # Assuming stoploss on exchnage is hit # stoploss_order_id should become None # and trade should be sold at the price of stoploss - stoploss_limit_executed = MagicMock(return_value={ + stoploss_executed = MagicMock(return_value={ "id": "123", "timestamp": 1542707426845, "datetime": "2018-11-20T09:50:26.845Z", @@ -2525,7 +2525,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f "fee": None, "trades": None }) - mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_limit_executed) + mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_executed) freqtrade.exit_positions(trades) assert trade.stoploss_order_id is None diff --git a/tests/test_integration.py b/tests/test_integration.py index 9cb071bb8..c40da7e9d 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -20,7 +20,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, default_conf['max_open_trades'] = 3 default_conf['exchange']['name'] = 'binance' - stoploss_limit = { + stoploss = { 'id': 123, 'info': {} } @@ -53,7 +53,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL)] ) cancel_order_mock = MagicMock() - mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, From e6f1912443fa4a1229ac53ee2f1af3f5be3804ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jan 2020 14:07:59 +0100 Subject: [PATCH 079/317] Use named arguments for stoploss create_order call --- freqtrade/exchange/binance.py | 4 ++-- tests/exchange/test_binance.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index d08726cf0..8a3e28379 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -64,8 +64,8 @@ class Binance(Exchange): rate = self.price_to_precision(pair, rate) - order = self._api.create_order(pair, ordertype, 'sell', - amount, rate, params) + order = self._api.create_order(symbol=pair, type=ordertype, side='sell', + amount=amount, price=stop_price, params=params) logger.info('stoploss limit order added for %s. ' 'stop price: %s. limit: %s', pair, stop_price, rate) return order diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index fdf3d7435..a1b24913e 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -9,7 +9,7 @@ from freqtrade.exceptions import (DependencyException, InvalidOrderException, from tests.conftest import get_patched_exchange -def test_stoploss_limit_order(default_conf, mocker): +def test_stoploss_order_binance(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) order_type = 'stop_loss_limit' @@ -38,11 +38,12 @@ def test_stoploss_limit_order(default_conf, mocker): assert 'id' in order assert 'info' in order assert order['id'] == order_id - assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' - assert api_mock.create_order.call_args[0][1] == order_type - assert api_mock.create_order.call_args[0][2] == 'sell' - assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} + assert api_mock.create_order.call_args_list[0][1]['symbol'] == 'ETH/BTC' + assert api_mock.create_order.call_args_list[0][1]['type'] == order_type + assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' + assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 + assert api_mock.create_order.call_args_list[0][1]['price'] == 220 + assert api_mock.create_order.call_args_list[0][1]['params'] == {'stopPrice': 220} # test exception handling with pytest.raises(DependencyException): @@ -67,7 +68,7 @@ def test_stoploss_limit_order(default_conf, mocker): exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) -def test_stoploss_limit_order_dry_run(default_conf, mocker): +def test_stoploss_order_dry_run_binance(default_conf, mocker): api_mock = MagicMock() order_type = 'stop_loss_limit' default_conf['dry_run'] = True From f1629c907a3ea88d08c2976223d25c2ce82e56f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jan 2020 14:08:47 +0100 Subject: [PATCH 080/317] Implement stoploss for kraken --- freqtrade/exchange/kraken.py | 45 +++++++++++++++++- tests/exchange/test_kraken.py | 87 +++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 9bcd9cc1f..88c414772 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -4,7 +4,8 @@ from typing import Dict import ccxt -from freqtrade.exceptions import OperationalException, TemporaryError +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.exchange import Exchange from freqtrade.exchange.exchange import retrier @@ -15,6 +16,7 @@ class Kraken(Exchange): _params: Dict = {"trading_agreement": "agree"} _ft_has: Dict = { + "stoploss_on_exchange": True, "trades_pagination": "id", "trades_pagination_arg": "since", } @@ -48,3 +50,44 @@ class Kraken(Exchange): f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e + + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: + """ + Creates a stoploss market order. + Stoploss market orders is the only stoploss type supported by kraken. + """ + + ordertype = "stop-loss" + + stop_price = self.price_to_precision(pair, stop_price) + + if self._config['dry_run']: + dry_order = self.dry_run_order( + pair, ordertype, "sell", amount, stop_price) + return dry_order + + try: + params = self._params.copy() + + amount = self.amount_to_precision(pair, amount) + + order = self._api.create_order(symbol=pair, type=ordertype, side='sell', + amount=amount, price=stop_price, params=params) + logger.info('stoploss order added for %s. ' + 'stop price: %s.', pair, stop_price) + return order + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to create {ordertype} sell order on market {pair}.' + f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not create {ordertype} sell order on market {pair}. ' + f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' + f'Message: {e}') from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py index 8490ee1a2..241d15772 100644 --- a/tests/exchange/test_kraken.py +++ b/tests/exchange/test_kraken.py @@ -3,6 +3,11 @@ from random import randint from unittest.mock import MagicMock +import ccxt +import pytest + +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from tests.conftest import get_patched_exchange from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -149,3 +154,85 @@ def test_get_balances_prod(default_conf, mocker): assert balances['4ST']['used'] == 0.0 ccxt_exceptionhandlers(mocker, default_conf, api_mock, "kraken", "get_balances", "fetch_balance") + + +def test_stoploss_order_kraken(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'stop-loss' + + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + + default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + + # stoploss_on_exchange_limit_ratio is irrelevant for kraken market orders + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=190, + order_types={'stoploss_on_exchange_limit_ratio': 1.05}) + assert api_mock.create_order.call_count == 1 + + api_mock.create_order.reset_mock() + + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args_list[0][1]['symbol'] == 'ETH/BTC' + assert api_mock.create_order.call_args_list[0][1]['type'] == order_type + assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' + assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 + assert api_mock.create_order.call_args_list[0][1]['price'] == 220 + assert api_mock.create_order.call_args_list[0][1]['params'] == {'trading_agreement': 'agree'} + + # test exception handling + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + with pytest.raises(InvalidOrderException): + api_mock.create_order = MagicMock( + side_effect=ccxt.InvalidOrder("kraken Order would trigger immediately.")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + with pytest.raises(TemporaryError): + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + with pytest.raises(OperationalException, match=r".*DeadBeef.*"): + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + +def test_stoploss_order_dry_run_kraken(default_conf, mocker): + api_mock = MagicMock() + order_type = 'stop-loss' + default_conf['dry_run'] = True + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + + api_mock.create_order.reset_mock() + + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + assert 'id' in order + assert 'info' in order + assert 'type' in order + + assert order['type'] == order_type + assert order['price'] == 220 + assert order['amount'] == 1 From 7a22aaa11144dc95cb1f855642fdac555d866537 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jan 2020 14:30:09 +0100 Subject: [PATCH 081/317] UPdate documentation to reflect that stoploss-on-exchange is also available for kraken --- docs/configuration.md | 2 +- docs/exchanges.md | 5 ++++- docs/stoploss.md | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index fe692eacb..f2d0fa5f2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -278,7 +278,7 @@ If this is configured, the following 4 values (`buy`, `sell`, `stoploss` and The below is the default which is used if this is not configured in either strategy or configuration file. Since `stoploss_on_exchange` uses limit orders, the exchange needs 2 prices, the stoploss_price and the Limit price. -`stoploss` defines the stop-price - and limit should be slightly below this. This defaults to 0.99 / 1%. +`stoploss` defines the stop-price - and limit should be slightly below this. This defaults to 0.99 / 1% (configurable via `stoploss_on_exchange_limit_ratio`). Calculation example: we bought the asset at 100$. Stop-price is 95$, then limit would be `95 * 0.99 = 94.05$` - so the stoploss will happen between 95$ and 94.05$. diff --git a/docs/exchanges.md b/docs/exchanges.md index 76fa81f4a..18a9f1cba 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -5,7 +5,7 @@ This page combines common gotchas and informations which are exchange-specific a ## Binance !!! Tip "Stoploss on Exchange" - Binance is currently the only exchange supporting `stoploss_on_exchange`. It provides great advantages, so we recommend to benefit from it. + Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. ### Blacklists @@ -22,6 +22,9 @@ Binance has been split into 3, and users must use the correct ccxt exchange ID f ## Kraken +!!! Tip "Stoploss on Exchange" + Kraken supports `stoploss_on_exchange` and uses stop-loss-market orders. It provides great advantages, so we recommend to benefit from it, however since the resulting order is a stoploss-market order, sell-rates are not guaranteed, which makes this feature less secure than on other exchanges. This limitation is based on kraken's policy [source](https://blog.kraken.com/post/1494/kraken-enables-advanced-orders-and-adds-10-currency-pairs/) and [source2](https://blog.kraken.com/post/1494/kraken-enables-advanced-orders-and-adds-10-currency-pairs/) - which has stoploss-limit orders disabled. + ### Historic Kraken data The Kraken API does only provide 720 historic candles, which is sufficient for Freqtrade dry-run and live trade modes, but is a problem for backtesting. diff --git a/docs/stoploss.md b/docs/stoploss.md index 105488296..f6d56fd41 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -27,7 +27,7 @@ So this parameter will tell the bot how often it should update the stoploss orde This same logic will reapply a stoploss order on the exchange should you cancel it accidentally. !!! Note - Stoploss on exchange is only supported for Binance as of now. + Stoploss on exchange is only supported for Binance (stop-loss-limit) and Kraken (stop-loss-market) as of now. ## Static Stop Loss From cf9331919fdf658ee86a0b21642c8c717cf0d1b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jan 2020 19:54:30 +0100 Subject: [PATCH 082/317] move exchange-specific order-parsing to exchange class Related to stoploss_on_exchange in combination with trailing stoploss. Binance contains stopPrice in the info, while kraken returns the same value as "price". --- freqtrade/exchange/binance.py | 7 +++++++ freqtrade/exchange/exchange.py | 13 ++++++++++--- freqtrade/exchange/kraken.py | 7 +++++++ freqtrade/freqtradebot.py | 3 +-- tests/exchange/test_binance.py | 14 ++++++++++++++ tests/exchange/test_exchange.py | 3 +++ tests/exchange/test_kraken.py | 13 +++++++++++++ 7 files changed, 55 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 8a3e28379..45102359d 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -32,6 +32,13 @@ class Binance(Exchange): return super().get_order_book(pair, limit) + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: + """ + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + return order['type'] == 'stop_loss_limit' and stop_loss > float(order['info']['stopPrice']) + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: """ creates a stoploss limit order. diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index bef92750c..a8df4c1bb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -282,8 +282,8 @@ class Exchange: quote_currencies = self.get_quote_currencies() if stake_currency not in quote_currencies: raise OperationalException( - f"{stake_currency} is not available as stake on {self.name}. " - f"Available currencies are: {', '.join(quote_currencies)}") + f"{stake_currency} is not available as stake on {self.name}. " + f"Available currencies are: {', '.join(quote_currencies)}") def validate_pairs(self, pairs: List[str]) -> None: """ @@ -460,7 +460,7 @@ class Exchange: "status": "closed", "filled": closed_order["amount"], "remaining": 0 - }) + }) if closed_order["type"] in ["stop_loss_limit"]: closed_order["info"].update({"stopPrice": closed_order["price"]}) self._dry_run_open_orders[closed_order["id"]] = closed_order @@ -519,6 +519,13 @@ class Exchange: return self.create_order(pair, ordertype, 'sell', amount, rate, params) + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: + """ + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + raise OperationalException(f"stoploss is not implemented for {self.name}.") + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: """ creates a stoploss order. diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 88c414772..243f1a6d6 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -51,6 +51,13 @@ class Kraken(Exchange): except ccxt.BaseError as e: raise OperationalException(e) from e + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: + """ + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + return order['type'] == 'stop-loss' and stop_loss > float(order['price']) + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: """ Creates a stoploss market order. diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a4b0ab806..fa9a8424a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -718,8 +718,7 @@ class FreqtradeBot: :param order: Current on exchange stoploss order :return: None """ - - if trade.stop_loss > float(order['info']['stopPrice']): + if self.exchange.stoploss_adjust(trade.stop_loss, order): # we check if the update is neccesary update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60) if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat: diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index a1b24913e..e4599dcd7 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -92,3 +92,17 @@ def test_stoploss_order_dry_run_binance(default_conf, mocker): assert order['type'] == order_type assert order['price'] == 220 assert order['amount'] == 1 + + +def test_stoploss_adjust_binance(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf, id='binance') + order = { + 'type': 'stop_loss_limit', + 'price': 1500, + 'info': {'stopPrice': 1500}, + } + assert exchange.stoploss_adjust(1501, order) + assert not exchange.stoploss_adjust(1499, order) + # Test with invalid order case + order['type'] = 'stop_loss' + assert not exchange.stoploss_adjust(1501, order) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 680e69764..3a664a9ec 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1763,6 +1763,9 @@ def test_stoploss_order_unsupported_exchange(default_conf, mocker): with pytest.raises(OperationalException, match=r"stoploss is not implemented .*"): exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + with pytest.raises(OperationalException, match=r"stoploss is not implemented .*"): + exchange.stoploss_adjust(1, {}) + def test_merge_ft_has_dict(default_conf, mocker): mocker.patch.multiple('freqtrade.exchange.Exchange', diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py index 241d15772..d63dd66cc 100644 --- a/tests/exchange/test_kraken.py +++ b/tests/exchange/test_kraken.py @@ -236,3 +236,16 @@ def test_stoploss_order_dry_run_kraken(default_conf, mocker): assert order['type'] == order_type assert order['price'] == 220 assert order['amount'] == 1 + + +def test_stoploss_adjust_kraken(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf, id='kraken') + order = { + 'type': 'stop-loss', + 'price': 1500, + } + assert exchange.stoploss_adjust(1501, order) + assert not exchange.stoploss_adjust(1499, order) + # Test with invalid order case ... + order['type'] = 'stop_loss_limit' + assert not exchange.stoploss_adjust(1501, order) From 10d9db72a851769c1996f132e170588b4f708ee2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Jan 2020 20:06:04 +0100 Subject: [PATCH 083/317] Adjust tests slightly --- tests/test_freqtradebot.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a33d47f34..48bd2deb5 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1241,7 +1241,8 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss=stoploss + stoploss=stoploss, + stoploss_adjust=MagicMock(return_value=True), ) # enabling TSL @@ -1335,7 +1336,8 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss=stoploss + stoploss=stoploss, + stoploss_adjust=MagicMock(return_value=True), ) # enabling TSL @@ -1396,6 +1398,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, patch_edge(mocker) edge_conf['max_open_trades'] = float('inf') edge_conf['dry_run_wallet'] = 999.9 + edge_conf['exchange']['name'] = 'binance' mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ @@ -1406,7 +1409,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss=stoploss + stoploss=stoploss, ) # enabling TSL @@ -1459,7 +1462,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock() mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order_mock) - mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss_order_mock) + mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss_order_mock) # price goes down 5% mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ From f5a44e4fc440ff4bedeb07617d33928d3017d858 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Jan 2020 19:38:35 +0100 Subject: [PATCH 084/317] open_order_id should be None when handling stoploss orders --- tests/test_freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e0f2ecd3a..a80bb7452 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1165,7 +1165,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, freqtrade.enter_positions() trade = Trade.query.first() trade.is_open = True - trade.open_order_id = '12345' + trade.open_order_id = None trade.stoploss_order_id = 100 assert trade From a83de241e41155dbef49ffd858e994033c5a1cfa Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Jan 2020 19:40:31 +0100 Subject: [PATCH 085/317] Check for closed stoploss-orders first --- freqtrade/freqtradebot.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e3856e200..5505005ff 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -679,6 +679,16 @@ class FreqtradeBot: except InvalidOrderException as exception: logger.warning('Unable to fetch stoploss order: %s', exception) + # We check if stoploss order is fulfilled + if stoploss_order and stoploss_order['status'] == 'closed': + trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value + trade.update(stoploss_order) + # Lock pair for one candle to prevent immediate rebuys + self.strategy.lock_pair(trade.pair, + timeframe_to_next_date(self.config['ticker_interval'])) + self._notify_sell(trade, "stoploss") + return True + # If buy order is fulfilled but there is no stoploss, we add a stoploss on exchange if (not trade.open_order_id and not stoploss_order): @@ -699,16 +709,6 @@ class FreqtradeBot: trade.stoploss_order_id = None logger.warning('Stoploss order was cancelled, but unable to recreate one.') - # We check if stoploss order is fulfilled - if stoploss_order and stoploss_order['status'] == 'closed': - trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value - trade.update(stoploss_order) - # Lock pair for one candle to prevent immediate rebuys - self.strategy.lock_pair(trade.pair, - timeframe_to_next_date(self.config['ticker_interval'])) - self._notify_sell(trade, "stoploss") - return True - # Finally we check if stoploss on exchange should be moved up because of trailing. if stoploss_order and self.config.get('trailing_stop', False): # if trailing stoploss is enabled we check if stoploss value has changed From ea5ac1efb531058cf4ed67ba4fccd4306cad8af5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Jan 2020 20:24:23 +0100 Subject: [PATCH 086/317] Don't handle stoploss if there is an open regular order --- freqtrade/freqtradebot.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5505005ff..c150d1aa9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -689,8 +689,13 @@ class FreqtradeBot: self._notify_sell(trade, "stoploss") return True + if trade.open_order_id: + # Trade has an open Buy or Sell order, Stoploss-handling can't happen in this case + # as the Amount on the exchange is tied up in another trade. + return False + # If buy order is fulfilled but there is no stoploss, we add a stoploss on exchange - if (not trade.open_order_id and not stoploss_order): + if (not stoploss_order): stoploss = self.edge.stoploss(pair=trade.pair) if self.edge else self.strategy.stoploss From 70b9bd9c0e7d2b37a4386f0809af7237a9dada6a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Jan 2020 20:36:48 +0100 Subject: [PATCH 087/317] Verify if trade is closed before acting on Stoploss_on_exchange --- freqtrade/freqtradebot.py | 3 ++- tests/test_freqtradebot.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c150d1aa9..9f06cbb67 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -689,9 +689,10 @@ class FreqtradeBot: self._notify_sell(trade, "stoploss") return True - if trade.open_order_id: + if trade.open_order_id or not trade.is_open: # Trade has an open Buy or Sell order, Stoploss-handling can't happen in this case # as the Amount on the exchange is tied up in another trade. + # The trade can be closed already (sell-order fill confirmation came in this iteration) return False # If buy order is fulfilled but there is no stoploss, we add a stoploss on exchange diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a80bb7452..65b5adda5 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1127,6 +1127,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, 'freqtrade.exchange.Exchange.stoploss_limit', side_effect=DependencyException() ) + trade.is_open = True freqtrade.handle_stoploss_on_exchange(trade) assert log_has('Unable to place a stoploss order on exchange.', caplog) assert trade.stoploss_order_id is None From 72c273aaedc237e70e049a67db4d469faf116c03 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Jan 2020 21:07:11 +0100 Subject: [PATCH 088/317] Add test for closed trade case --- tests/test_freqtradebot.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 65b5adda5..147ad9d7c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1141,6 +1141,16 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, freqtrade.handle_stoploss_on_exchange(trade) assert stoploss_limit.call_count == 1 + # Sixth case: Closed Trade + # Should not create new order + trade.stoploss_order_id = None + trade.is_open = False + stoploss_limit.reset_mock() + mocker.patch('freqtrade.exchange.Exchange.get_order') + mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + assert freqtrade.handle_stoploss_on_exchange(trade) is False + assert stoploss_limit.call_count == 0 + def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, limit_buy_order, limit_sell_order) -> None: From f8db7f170981898fcc7509b59acdb25b1f01550b Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 25 Jan 2020 04:17:41 +0100 Subject: [PATCH 089/317] added ask price, bid price, immediate ask quantity, and immediate bid quantity to check_depth_of_market_buy. also added a line that mentions if delta condition was satisfied or not. --- freqtrade/freqtradebot.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e3856e200..f1584c731 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -423,11 +423,14 @@ class FreqtradeBot: order_book_bids = order_book_data_frame['b_size'].sum() order_book_asks = order_book_data_frame['a_size'].sum() bids_ask_delta = order_book_bids / order_book_asks - logger.info('bids: %s, asks: %s, delta: %s', order_book_bids, - order_book_asks, bids_ask_delta) + logger.info('bids: %s, asks: %s, delta: %s, askprice: %s, bidprice: %s, immediate askquantity: %s, immediate bidquantity: %s', + order_book_bids, order_book_asks, bids_ask_delta, order_book['asks'][0][0], order_book['bids'][0][0], order_book['asks'][0][1], order_book['bids'][0][1]) if bids_ask_delta >= conf_bids_to_ask_delta: + logger.info('bids to ask delta DOES satisfy condition.') return True - return False + else: + logger.info('bids to ask delta DOES NOT satisfy condition.') + return False def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None) -> bool: """ From 4c0e586354f3f4cd87138c51ee695136d45f8caf Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 27 Jan 2020 22:39:04 +0300 Subject: [PATCH 090/317] Advise to use https method for git clone i.o ssh --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 267d91c8d..cbe000da4 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -42,7 +42,7 @@ The easiest way to install and run Freqtrade is to clone the bot GitHub reposito This can be achieved with the following commands: ```bash -git clone git@github.com:freqtrade/freqtrade.git +git clone https://github.com/freqtrade/freqtrade.git cd freqtrade git checkout master # Optional, see (1) ./setup.sh --install From 328a9ffafdad4953c71fb20f4bd8b18251f3f737 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 25 Jan 2020 20:53:02 +0100 Subject: [PATCH 091/317] fixed typo in false statement --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f1584c731..323d4d14c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -429,7 +429,7 @@ class FreqtradeBot: logger.info('bids to ask delta DOES satisfy condition.') return True else: - logger.info('bids to ask delta DOES NOT satisfy condition.') + logger.info(f"bids to ask delta for {pair} does not satisfy condition.") return False def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None) -> bool: From a0b92fe0b12f26514e3b623eb91f44022604e67f Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 28 Jan 2020 17:09:44 +0100 Subject: [PATCH 092/317] removed typo --- freqtrade/freqtradebot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 323d4d14c..9df2acaf0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -423,8 +423,12 @@ class FreqtradeBot: order_book_bids = order_book_data_frame['b_size'].sum() order_book_asks = order_book_data_frame['a_size'].sum() bids_ask_delta = order_book_bids / order_book_asks - logger.info('bids: %s, asks: %s, delta: %s, askprice: %s, bidprice: %s, immediate askquantity: %s, immediate bidquantity: %s', - order_book_bids, order_book_asks, bids_ask_delta, order_book['asks'][0][0], order_book['bids'][0][0], order_book['asks'][0][1], order_book['bids'][0][1]) + logger.info( + f"bids: {order_book_bids}, asks: {order_book_asks}, delta: {bids_ask_delta}, " + f"askprice: {order_book['asks'][0][0]}, bidprice: {order_book['bids'][0][0]}, " + f"immediate ask quantity: {order_book['asks'][0][1]}, " + f"immediate bid quantity: {order_book['bids'][0][1]}", + ) if bids_ask_delta >= conf_bids_to_ask_delta: logger.info('bids to ask delta DOES satisfy condition.') return True From b384ca8fd28bfc41226718451f9dfcc58a0ce70b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 Jan 2020 20:30:03 +0100 Subject: [PATCH 093/317] Create new-config command --- freqtrade/commands/__init__.py | 1 + freqtrade/commands/arguments.py | 13 +++++++++++-- freqtrade/commands/deploy_commands.py | 8 ++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 990c1107a..81467cf61 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -9,6 +9,7 @@ Note: Be careful with file-scoped imports in these subfiles. from freqtrade.commands.arguments import Arguments from freqtrade.commands.data_commands import start_download_data from freqtrade.commands.deploy_commands import (start_create_userdir, + start_new_config, start_new_hyperopt, start_new_strategy) from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 724814554..504c6b0b5 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -43,6 +43,8 @@ ARGS_TEST_PAIRLIST = ["config", "quote_currencies", "print_one_column", "list_pa ARGS_CREATE_USERDIR = ["user_data_dir", "reset"] +ARGS_BUILD_CONFIG = ["config"] + ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] @@ -133,8 +135,9 @@ class Arguments: from freqtrade.commands import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, - start_list_strategies, start_new_hyperopt, - start_new_strategy, start_list_timeframes, + start_list_strategies, start_list_timeframes, + start_new_config, + start_new_hyperopt, start_new_strategy, start_plot_dataframe, start_plot_profit, start_backtesting, start_hyperopt, start_edge, start_test_pairlist, start_trading) @@ -177,6 +180,12 @@ class Arguments: create_userdir_cmd.set_defaults(func=start_create_userdir) self._build_args(optionlist=ARGS_CREATE_USERDIR, parser=create_userdir_cmd) + # add new-config subcommand + build_config_cmd = subparsers.add_parser('new-config', + help="Create new config") + build_config_cmd.set_defaults(func=start_new_config) + self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd) + # add new-strategy subcommand build_strategy_cmd = subparsers.add_parser('new-strategy', help="Create new strategy") diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 99ae63244..34755932c 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -110,3 +110,11 @@ def start_new_hyperopt(args: Dict[str, Any]) -> None: deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) else: raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") + + +def start_new_config(args: Dict[str, Any]) -> None: + """ + Create a new strategy from a template + Asking the user questions to fill out the templateaccordingly. + """ + pass From 9f291282056fde93c3cab74470764a9cdf80f89b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 07:01:17 +0100 Subject: [PATCH 094/317] Fix small json formatting issue --- config.json.example | 2 +- config_binance.json.example | 2 +- config_full.json.example | 2 +- config_kraken.json.example | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config.json.example b/config.json.example index 8b85e71eb..a03ab6c2a 100644 --- a/config.json.example +++ b/config.json.example @@ -4,7 +4,7 @@ "stake_amount": 0.05, "tradable_balance_ratio": 0.99, "fiat_display_currency": "USD", - "ticker_interval" : "5m", + "ticker_interval": "5m", "dry_run": false, "trailing_stop": false, "unfilledtimeout": { diff --git a/config_binance.json.example b/config_binance.json.example index 0521a3a35..e2c9879b0 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -4,7 +4,7 @@ "stake_amount": 0.05, "tradable_balance_ratio": 0.99, "fiat_display_currency": "USD", - "ticker_interval" : "5m", + "ticker_interval": "5m", "dry_run": true, "trailing_stop": false, "unfilledtimeout": { diff --git a/config_full.json.example b/config_full.json.example index 82d8bd04a..f543604e7 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -4,7 +4,7 @@ "stake_amount": 0.05, "tradable_balance_ratio": 0.99, "fiat_display_currency": "USD", - "amount_reserve_percent" : 0.05, + "amount_reserve_percent": 0.05, "amend_last_stake_amount": false, "last_stake_amount_min_ratio": 0.5, "dry_run": false, diff --git a/config_kraken.json.example b/config_kraken.json.example index a527b569d..4f74d0b7d 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -4,7 +4,7 @@ "stake_amount": 10, "tradable_balance_ratio": 0.99, "fiat_display_currency": "EUR", - "ticker_interval" : "5m", + "ticker_interval": "5m", "dry_run": true, "trailing_stop": false, "unfilledtimeout": { From 122c9163566acfbbf2d2a18e86dfcfac3a772dab Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 07:01:32 +0100 Subject: [PATCH 095/317] Add first version of config_deploy --- freqtrade/commands/deploy_commands.py | 28 ++++++++- freqtrade/templates/base_config.json.j2 | 82 +++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 freqtrade/templates/base_config.json.j2 diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 34755932c..4e114a6d8 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -112,9 +112,35 @@ def start_new_hyperopt(args: Dict[str, Any]) -> None: raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") +def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: + """ + Applies selections to the template and writes the result to config_path + :param config_path: Path object for new config file. Should not exist yet + :param selecions: Dict containing selections taken by the user. + """ + config_text = render_template(templatefile='base_config.json.j2', + arguments=selections) + + config_path.write_text(config_text) + + def start_new_config(args: Dict[str, Any]) -> None: """ Create a new strategy from a template Asking the user questions to fill out the templateaccordingly. """ - pass + sample_selections = { + 'stake_currency': 'USDT', + 'stake_amount': 100, + 'fiat_display_currency': 'EUR', + 'ticker_interval': '15m', + 'dry_run': True, + 'exchange': 'binance', + 'exchange_key': 'sampleKey', + 'exchange_secret': 'Samplesecret', + 'telegram': True, + 'telegram_token': 'asdf1244', + 'telegram_chat_id': '1144444', + } + config_path = Path(args['config'][0]) + deploy_new_config(config_path, sample_selections) diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 new file mode 100644 index 000000000..ad8a762ec --- /dev/null +++ b/freqtrade/templates/base_config.json.j2 @@ -0,0 +1,82 @@ +{ + "max_open_trades": 3, + "stake_currency": "{{ stake_currency }}", + "stake_amount": {{ stake_amount }}, + "tradable_balance_ratio": 0.99, + "fiat_display_currency": "{{ fiat_display_currency }}", + "ticker_interval": "{{ ticker_interval }}", + "dry_run": {{ dry_run | lower }}, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, + "bid_strategy": { + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9, + "use_sell_signal": true, + "sell_profit_only": false, + "ignore_roi_if_buy_signal": false + }, + "exchange": { + "name": "bittrex", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 500 + }, + "pair_whitelist": [ + "ETH/BTC", + "LTC/BTC", + "ETC/BTC", + "DASH/BTC", + "ZEC/BTC", + "XLM/BTC", + "NXT/BTC", + "TRX/BTC", + "ADA/BTC", + "XMR/BTC" + ], + "pair_blacklist": [ + "DOGE/BTC" + ] + }, + "pairlists": [ + {"method": "StaticPairList"} + ], + "edge": { + "enabled": false, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 7, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, + "minimum_expectancy": 0.20, + "min_trade_number": 10, + "max_trade_duration_minute": 1440, + "remove_pumps": false + }, + "telegram": { + "enabled": {{ telegram | lower }}, + "token": "{{ telegram_token }}", + "chat_id": "{{ telegram_chat_id }}" + }, + "initial_state": "running", + "forcebuy_enable": false, + "internals": { + "process_throttle_secs": 5 + } +} From c80d8f432acc07665353947acae84a4d09cd3ced Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 07:13:38 +0100 Subject: [PATCH 096/317] Add exchange templates --- freqtrade/commands/deploy_commands.py | 19 +++++++++-- freqtrade/misc.py | 1 - freqtrade/templates/base_config.json.j2 | 24 +------------- .../subtemplates/exchange_binance.j2 | 28 ++++++++++++++++ .../subtemplates/exchange_generic.j2 | 13 ++++++++ .../templates/subtemplates/exchange_kraken.j2 | 33 +++++++++++++++++++ 6 files changed, 92 insertions(+), 26 deletions(-) create mode 100644 freqtrade/templates/subtemplates/exchange_binance.j2 create mode 100644 freqtrade/templates/subtemplates/exchange_generic.j2 create mode 100644 freqtrade/templates/subtemplates/exchange_kraken.j2 diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 4e114a6d8..065703faa 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -118,9 +118,22 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: :param config_path: Path object for new config file. Should not exist yet :param selecions: Dict containing selections taken by the user. """ + from jinja2.exceptions import TemplateNotFound + try: + selections['exchange'] = render_template( + templatefile=f"subtemplates/exchange_{selections['exchange_name']}.j2", + arguments=selections + ) + except TemplateNotFound: + selections['exchange'] = render_template( + templatefile=f"subtemplates/exchange_generic.j2", + arguments=selections + ) + config_text = render_template(templatefile='base_config.json.j2', arguments=selections) + logger.info(f"Writing config to `{config_path}`.") config_path.write_text(config_text) @@ -135,12 +148,14 @@ def start_new_config(args: Dict[str, Any]) -> None: 'fiat_display_currency': 'EUR', 'ticker_interval': '15m', 'dry_run': True, - 'exchange': 'binance', + 'exchange_name': 'binance', 'exchange_key': 'sampleKey', 'exchange_secret': 'Samplesecret', - 'telegram': True, + 'telegram': False, 'telegram_token': 'asdf1244', 'telegram_chat_id': '1144444', } config_path = Path(args['config'][0]) deploy_new_config(config_path, sample_selections) + + diff --git a/freqtrade/misc.py b/freqtrade/misc.py index bcba78cf0..40e1fdf17 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -138,5 +138,4 @@ def render_template(templatefile: str, arguments: dict = {}): autoescape=select_autoescape(['html', 'xml']) ) template = env.get_template(templatefile) - return template.render(**arguments) diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index ad8a762ec..f2f919e1f 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -28,29 +28,7 @@ "ignore_roi_if_buy_signal": false }, "exchange": { - "name": "bittrex", - "key": "your_exchange_key", - "secret": "your_exchange_secret", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 500 - }, - "pair_whitelist": [ - "ETH/BTC", - "LTC/BTC", - "ETC/BTC", - "DASH/BTC", - "ZEC/BTC", - "XLM/BTC", - "NXT/BTC", - "TRX/BTC", - "ADA/BTC", - "XMR/BTC" - ], - "pair_blacklist": [ - "DOGE/BTC" - ] + {{ exchange | indent(8) }} }, "pairlists": [ {"method": "StaticPairList"} diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 new file mode 100644 index 000000000..082af45c4 --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -0,0 +1,28 @@ +"name": "{{ exchange_name | lower }}", +"key": "{{ exchange_key }}", +"secret": "{{ exchange_secret }}", +"ccxt_config": {"enableRateLimit": true}, +"ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 200 +}, +"pair_whitelist": [ + "ALGO/BTC", + "ATOM/BTC", + "BAT/BTC", + "BCH/BTC", + "BRD/BTC", + "EOS/BTC", + "ETH/BTC", + "IOTA/BTC", + "LINK/BTC", + "LTC/BTC", + "NEO/BTC", + "NXS/BTC", + "XMR/BTC", + "XRP/BTC", + "XTZ/BTC" +], +"pair_blacklist": [ + "BNB/BTC" +] diff --git a/freqtrade/templates/subtemplates/exchange_generic.j2 b/freqtrade/templates/subtemplates/exchange_generic.j2 new file mode 100644 index 000000000..5d5bee2b2 --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_generic.j2 @@ -0,0 +1,13 @@ +"name": "{{ exchange_name | lower }}", +"key": "{{ exchange_key }}", +"secret": "{{ exchange_secret }}", +"ccxt_config": {"enableRateLimit": true}, +"ccxt_async_config": { + "enableRateLimit": true, +}, +"pair_whitelist": [ + +], +"pair_blacklist": [ + +] diff --git a/freqtrade/templates/subtemplates/exchange_kraken.j2 b/freqtrade/templates/subtemplates/exchange_kraken.j2 new file mode 100644 index 000000000..690828887 --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_kraken.j2 @@ -0,0 +1,33 @@ +"name": "kraken", +"key": "{{ exchange_key }}", +"secret": "{{ exchange_secret }}", +"ccxt_config": {"enableRateLimit": true}, +"ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 1000 +}, +"pair_whitelist": [ + "ADA/EUR", + "ATOM/EUR", + "BAT/EUR", + "BCH/EUR", + "BTC/EUR", + "DAI/EUR", + "DASH/EUR", + "EOS/EUR", + "ETC/EUR", + "ETH/EUR", + "LINK/EUR", + "LTC/EUR", + "QTUM/EUR", + "REP/EUR", + "WAVES/EUR", + "XLM/EUR", + "XMR/EUR", + "XRP/EUR", + "XTZ/EUR", + "ZEC/EUR" +], +"pair_blacklist": [ + +] From 68771a78617b6ff4af6baca08df1f54d76ba9338 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 29 Jan 2020 17:08:36 +0300 Subject: [PATCH 097/317] Remove state attr from Worker --- freqtrade/worker.py | 8 -------- tests/test_freqtradebot.py | 6 +++--- tests/test_worker.py | 6 +++--- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 972ff0d61..6da04b4a2 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -56,14 +56,6 @@ class Worker: self._sd_notify = sdnotify.SystemdNotifier() if \ self._config.get('internals', {}).get('sd_notify', False) else None - @property - def state(self) -> State: - return self.freqtrade.state - - @state.setter - def state(self, value: State) -> None: - self.freqtrade.state = value - def run(self) -> None: state = None while True: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e0f2ecd3a..128d9c9ee 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -797,10 +797,10 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None: worker = Worker(args=None, config=default_conf) patch_get_signal(worker.freqtrade) - assert worker.state == State.RUNNING + assert worker.freqtrade.state == State.RUNNING worker._process() - assert worker.state == State.STOPPED + assert worker.freqtrade.state == State.STOPPED assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] @@ -3631,7 +3631,7 @@ def test_startup_state(default_conf, mocker): } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) worker = get_patched_worker(mocker, default_conf) - assert worker.state is State.RUNNING + assert worker.freqtrade.state is State.RUNNING def test_startup_trade_reinit(default_conf, edge_conf, mocker): diff --git a/tests/test_worker.py b/tests/test_worker.py index 72e215210..2fb42d47e 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -11,11 +11,11 @@ from tests.conftest import get_patched_worker, log_has def test_worker_state(mocker, default_conf, markets) -> None: mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) worker = get_patched_worker(mocker, default_conf) - assert worker.state is State.RUNNING + assert worker.freqtrade.state is State.RUNNING default_conf.pop('initial_state') worker = Worker(args=None, config=default_conf) - assert worker.state is State.STOPPED + assert worker.freqtrade.state is State.STOPPED def test_worker_running(mocker, default_conf, caplog) -> None: @@ -41,7 +41,7 @@ def test_worker_stopped(mocker, default_conf, caplog) -> None: mock_sleep = mocker.patch('time.sleep', return_value=None) worker = get_patched_worker(mocker, default_conf) - worker.state = State.STOPPED + worker.freqtrade.state = State.STOPPED state = worker._worker(old_state=State.RUNNING) assert state is State.STOPPED assert log_has('Changing state to: STOPPED', caplog) From dd83cb1b95fb2a0f3790d7c83a377ed72b659fd3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 20:27:38 +0100 Subject: [PATCH 098/317] Extract selection generation to a seperate method --- freqtrade/commands/deploy_commands.py | 38 ++++++++++++++++--------- freqtrade/templates/base_config.json.j2 | 2 +- tests/commands/test_commands.py | 21 ++++++++++++-- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 065703faa..87aea7492 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -112,6 +112,28 @@ def start_new_hyperopt(args: Dict[str, Any]) -> None: raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") +def ask_user_config() -> Dict[str, Any]: + """ + Ask user a few questions to build the configuration. + :returns: Dict with keys to put into template + """ + sample_selections = { + 'max_open_trades': 3, + 'stake_currency': 'USDT', + 'stake_amount': 100, + 'fiat_display_currency': 'EUR', + 'ticker_interval': '15m', + 'dry_run': True, + 'exchange_name': 'binance', + 'exchange_key': 'sampleKey', + 'exchange_secret': 'Samplesecret', + 'telegram': False, + 'telegram_token': 'asdf1244', + 'telegram_chat_id': '1144444', + } + return sample_selections + + def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: """ Applies selections to the template and writes the result to config_path @@ -142,20 +164,8 @@ def start_new_config(args: Dict[str, Any]) -> None: Create a new strategy from a template Asking the user questions to fill out the templateaccordingly. """ - sample_selections = { - 'stake_currency': 'USDT', - 'stake_amount': 100, - 'fiat_display_currency': 'EUR', - 'ticker_interval': '15m', - 'dry_run': True, - 'exchange_name': 'binance', - 'exchange_key': 'sampleKey', - 'exchange_secret': 'Samplesecret', - 'telegram': False, - 'telegram_token': 'asdf1244', - 'telegram_chat_id': '1144444', - } + selections = ask_user_config() config_path = Path(args['config'][0]) - deploy_new_config(config_path, sample_selections) + deploy_new_config(config_path, selections) diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index f2f919e1f..1370bfa80 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -1,5 +1,5 @@ { - "max_open_trades": 3, + "max_open_trades": {{ max_open_trades }}, "stake_currency": "{{ stake_currency }}", "stake_amount": {{ stake_amount }}, "tradable_balance_ratio": 0.99, diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 65d7f6eaf..f8efdfd3c 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -8,8 +8,9 @@ from freqtrade.commands import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, start_list_strategies, start_list_timeframes, - start_new_hyperopt, start_new_strategy, - start_test_pairlist, start_trading) + start_new_config, start_new_hyperopt, + start_new_strategy, start_test_pairlist, + start_trading) from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode @@ -537,6 +538,22 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): start_new_hyperopt(get_args(args)) +def test_start_new_config(mocker, caplog): + wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + + args = [ + "new-config", + "--config", + "coolconfig.json" + ] + start_new_config(get_args(args)) + + assert wt_mock.call_count == 1 + assert "binance" in wt_mock.call_args_list[0][0][0] + assert log_has_re("Writing config to .*", caplog) + + def test_download_data_keyboardInterrupt(mocker, caplog, markets): dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) From 49c9258a088e42db46656b24fc90d1227d604dc1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 20:32:27 +0100 Subject: [PATCH 099/317] enhance test --- tests/commands/test_commands.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index f8efdfd3c..19999f319 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1,3 +1,4 @@ +import json import re from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -538,10 +539,26 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): start_new_hyperopt(get_args(args)) -def test_start_new_config(mocker, caplog): +@pytest.mark.parametrize('exchange', ['bittrex', 'binance', 'kraken', 'ftx']) +def test_start_new_config(mocker, caplog, exchange): wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - + sample_selections = { + 'max_open_trades': 3, + 'stake_currency': 'USDT', + 'stake_amount': 100, + 'fiat_display_currency': 'EUR', + 'ticker_interval': '15m', + 'dry_run': True, + 'exchange_name': exchange, + 'exchange_key': 'sampleKey', + 'exchange_secret': 'Samplesecret', + 'telegram': False, + 'telegram_token': 'asdf1244', + 'telegram_chat_id': '1144444', + } + mocker.patch('freqtrade.commands.deploy_commands.ask_user_config', + return_value=sample_selections) args = [ "new-config", "--config", @@ -549,9 +566,11 @@ def test_start_new_config(mocker, caplog): ] start_new_config(get_args(args)) - assert wt_mock.call_count == 1 - assert "binance" in wt_mock.call_args_list[0][0][0] assert log_has_re("Writing config to .*", caplog) + assert wt_mock.call_count == 1 + result = json.loads(wt_mock.call_args_list[0][0][0]) + assert result['exchange']['name'] == exchange + assert result['ticker_interval'] == '15m' def test_download_data_keyboardInterrupt(mocker, caplog, markets): From e250c56829e74445e6177d4b61c17e57a0af21e0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 21:21:38 +0100 Subject: [PATCH 100/317] Add Questionaire workflow --- freqtrade/commands/deploy_commands.py | 98 ++++++++++++++++++++++++++- requirements-common.txt | 3 + setup.py | 2 + 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 87aea7492..670f272ce 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -3,11 +3,14 @@ import sys from pathlib import Path from typing import Any, Dict +from questionary import Separator, prompt + from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY from freqtrade.exceptions import OperationalException +from freqtrade.exchange import available_exchanges from freqtrade.misc import render_template from freqtrade.state import RunMode @@ -117,6 +120,99 @@ def ask_user_config() -> Dict[str, Any]: Ask user a few questions to build the configuration. :returns: Dict with keys to put into template """ + questions = [ + { + "type": "confirm", + "name": "dry_run", + "message": "Do you want to enable Dry-run (simulated trades)?", + "default": True, + }, + { + "type": "text", + "name": "stake_currency", + "message": "Please insert your stake currency:", + "default": 'BTC', + }, + { + "type": "text", + "name": "stake_amount", + "message": "Please insert your stake amount:", + "default": "0.01", + }, + { + "type": "text", + "name": "max_open_trades", + "message": "Please insert max_open_trades:", + "default": "3", + }, + { + "type": "text", + "name": "ticker_interval", + "message": "Please insert your ticker interval:", + "default": "5m", + }, + { + "type": "text", + "name": "fiat_display_currency", + "message": "Please insert your display Currency (for reporting):", + "default": 'USD', + }, + { + "type": "select", + "name": "exchange_name", + "message": "Select exchange", + "choices": [ + "bittrex", + "binance", + "binanceje", + "binanceus", + "kraken", + Separator(), + "other", + ], + }, + { + "type": "autocomplete", + "name": "exchange_name", + "message": "Type your exchange name (Must be supported by ccxt)", + "choices": available_exchanges(), + "when": lambda x: x["exchange_name"] == 'other' + }, + { + "type": "password", + "name": "exchange_key", + "message": "Insert Exchange Key", + "when": lambda x: not x['dry_run'] + }, + { + "type": "password", + "name": "exchange_secret", + "message": "Insert Exchange Secret", + "when": lambda x: not x['dry_run'] + }, + { + "type": "confirm", + "name": "telegram", + "message": "Do you want to enable Telegram?", + "default": False, + }, + { + "type": "password", + "name": "telegram_token", + "message": "Insert Telegram token", + "when": lambda x: x['telegram'] + }, + { + "type": "text", + "name": "telegram_chat_id", + "message": "Insert Telegram chat id", + "when": lambda x: x['telegram'] + }, + ] + answers = prompt(questions) + + print(answers) + sample_selections = { 'max_open_trades': 3, 'stake_currency': 'USDT', @@ -167,5 +263,3 @@ def start_new_config(args: Dict[str, Any]) -> None: selections = ask_user_config() config_path = Path(args['config'][0]) deploy_new_config(config_path, selections) - - diff --git a/requirements-common.txt b/requirements-common.txt index e4fe54721..80f18892b 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -28,3 +28,6 @@ flask==1.1.1 # Support for colorized terminal output colorama==0.4.3 +# Building config files interactively +questionary==1.5.1 +prompt-toolkit==3.0.3 diff --git a/setup.py b/setup.py index 7d8d7b68d..63a595f32 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,8 @@ setup(name='freqtrade', 'sdnotify', 'colorama', 'jinja2', + 'questionary', + 'prompt-toolkit', # from requirements.txt 'numpy', 'pandas', From 940bfbee96eef00e56d7f5097f1c59099b34bb37 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 21:28:01 +0100 Subject: [PATCH 101/317] Move start_config out of build_commands file --- freqtrade/commands/__init__.py | 2 +- freqtrade/commands/build_config_commands.py | 160 ++++++++++++++++++ freqtrade/commands/deploy_commands.py | 153 ----------------- .../subtemplates/exchange_generic.j2 | 2 +- tests/commands/test_commands.py | 2 +- 5 files changed, 163 insertions(+), 156 deletions(-) create mode 100644 freqtrade/commands/build_config_commands.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 81467cf61..6ea325e63 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -7,9 +7,9 @@ Note: Be careful with file-scoped imports in these subfiles. as they are parsed on startup, nothing containing optional modules should be loaded. """ from freqtrade.commands.arguments import Arguments +from freqtrade.commands.build_config_commands import start_new_config from freqtrade.commands.data_commands import start_download_data from freqtrade.commands.deploy_commands import (start_create_userdir, - start_new_config, start_new_hyperopt, start_new_strategy) from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py new file mode 100644 index 000000000..393416f53 --- /dev/null +++ b/freqtrade/commands/build_config_commands.py @@ -0,0 +1,160 @@ +import logging +from pathlib import Path +from typing import Any, Dict + +from questionary import Separator, prompt + +from freqtrade.exchange import available_exchanges +from freqtrade.misc import render_template + +logger = logging.getLogger(__name__) + + +def ask_user_config() -> Dict[str, Any]: + """ + Ask user a few questions to build the configuration. + :returns: Dict with keys to put into template + """ + questions = [ + { + "type": "confirm", + "name": "dry_run", + "message": "Do you want to enable Dry-run (simulated trades)?", + "default": True, + }, + { + "type": "text", + "name": "stake_currency", + "message": "Please insert your stake currency:", + "default": 'BTC', + }, + { + "type": "text", + "name": "stake_amount", + "message": "Please insert your stake amount:", + "default": "0.01", + }, + { + "type": "text", + "name": "max_open_trades", + "message": "Please insert max_open_trades:", + "default": "3", + }, + { + "type": "text", + "name": "ticker_interval", + "message": "Please insert your ticker interval:", + "default": "5m", + }, + { + "type": "text", + "name": "fiat_display_currency", + "message": "Please insert your display Currency (for reporting):", + "default": 'USD', + }, + { + "type": "select", + "name": "exchange_name", + "message": "Select exchange", + "choices": [ + "bittrex", + "binance", + "binanceje", + "binanceus", + "kraken", + Separator(), + "other", + ], + }, + { + "type": "autocomplete", + "name": "exchange_name", + "message": "Type your exchange name (Must be supported by ccxt)", + "choices": available_exchanges(), + "when": lambda x: x["exchange_name"] == 'other' + }, + { + "type": "password", + "name": "exchange_key", + "message": "Insert Exchange Key", + "when": lambda x: not x['dry_run'] + }, + { + "type": "password", + "name": "exchange_secret", + "message": "Insert Exchange Secret", + "when": lambda x: not x['dry_run'] + }, + { + "type": "confirm", + "name": "telegram", + "message": "Do you want to enable Telegram?", + "default": False, + }, + { + "type": "password", + "name": "telegram_token", + "message": "Insert Telegram token", + "when": lambda x: x['telegram'] + }, + { + "type": "text", + "name": "telegram_chat_id", + "message": "Insert Telegram chat id", + "when": lambda x: x['telegram'] + }, + ] + answers = prompt(questions) + + print(answers) + + sample_selections = { + 'max_open_trades': 3, + 'stake_currency': 'USDT', + 'stake_amount': 100, + 'fiat_display_currency': 'EUR', + 'ticker_interval': '15m', + 'dry_run': True, + 'exchange_name': 'binance', + 'exchange_key': 'sampleKey', + 'exchange_secret': 'Samplesecret', + 'telegram': False, + 'telegram_token': 'asdf1244', + 'telegram_chat_id': '1144444', + } + return sample_selections + + +def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: + """ + Applies selections to the template and writes the result to config_path + :param config_path: Path object for new config file. Should not exist yet + :param selecions: Dict containing selections taken by the user. + """ + from jinja2.exceptions import TemplateNotFound + try: + selections['exchange'] = render_template( + templatefile=f"subtemplates/exchange_{selections['exchange_name']}.j2", + arguments=selections + ) + except TemplateNotFound: + selections['exchange'] = render_template( + templatefile=f"subtemplates/exchange_generic.j2", + arguments=selections + ) + + config_text = render_template(templatefile='base_config.json.j2', + arguments=selections) + + logger.info(f"Writing config to `{config_path}`.") + config_path.write_text(config_text) + + +def start_new_config(args: Dict[str, Any]) -> None: + """ + Create a new strategy from a template + Asking the user questions to fill out the templateaccordingly. + """ + selections = ask_user_config() + config_path = Path(args['config'][0]) + deploy_new_config(config_path, selections) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 670f272ce..99ae63244 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -3,14 +3,11 @@ import sys from pathlib import Path from typing import Any, Dict -from questionary import Separator, prompt - from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY from freqtrade.exceptions import OperationalException -from freqtrade.exchange import available_exchanges from freqtrade.misc import render_template from freqtrade.state import RunMode @@ -113,153 +110,3 @@ def start_new_hyperopt(args: Dict[str, Any]) -> None: deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) else: raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") - - -def ask_user_config() -> Dict[str, Any]: - """ - Ask user a few questions to build the configuration. - :returns: Dict with keys to put into template - """ - questions = [ - { - "type": "confirm", - "name": "dry_run", - "message": "Do you want to enable Dry-run (simulated trades)?", - "default": True, - }, - { - "type": "text", - "name": "stake_currency", - "message": "Please insert your stake currency:", - "default": 'BTC', - }, - { - "type": "text", - "name": "stake_amount", - "message": "Please insert your stake amount:", - "default": "0.01", - }, - { - "type": "text", - "name": "max_open_trades", - "message": "Please insert max_open_trades:", - "default": "3", - }, - { - "type": "text", - "name": "ticker_interval", - "message": "Please insert your ticker interval:", - "default": "5m", - }, - { - "type": "text", - "name": "fiat_display_currency", - "message": "Please insert your display Currency (for reporting):", - "default": 'USD', - }, - { - "type": "select", - "name": "exchange_name", - "message": "Select exchange", - "choices": [ - "bittrex", - "binance", - "binanceje", - "binanceus", - "kraken", - Separator(), - "other", - ], - }, - { - "type": "autocomplete", - "name": "exchange_name", - "message": "Type your exchange name (Must be supported by ccxt)", - "choices": available_exchanges(), - "when": lambda x: x["exchange_name"] == 'other' - }, - { - "type": "password", - "name": "exchange_key", - "message": "Insert Exchange Key", - "when": lambda x: not x['dry_run'] - }, - { - "type": "password", - "name": "exchange_secret", - "message": "Insert Exchange Secret", - "when": lambda x: not x['dry_run'] - }, - { - "type": "confirm", - "name": "telegram", - "message": "Do you want to enable Telegram?", - "default": False, - }, - { - "type": "password", - "name": "telegram_token", - "message": "Insert Telegram token", - "when": lambda x: x['telegram'] - }, - { - "type": "text", - "name": "telegram_chat_id", - "message": "Insert Telegram chat id", - "when": lambda x: x['telegram'] - }, - ] - answers = prompt(questions) - - print(answers) - - sample_selections = { - 'max_open_trades': 3, - 'stake_currency': 'USDT', - 'stake_amount': 100, - 'fiat_display_currency': 'EUR', - 'ticker_interval': '15m', - 'dry_run': True, - 'exchange_name': 'binance', - 'exchange_key': 'sampleKey', - 'exchange_secret': 'Samplesecret', - 'telegram': False, - 'telegram_token': 'asdf1244', - 'telegram_chat_id': '1144444', - } - return sample_selections - - -def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: - """ - Applies selections to the template and writes the result to config_path - :param config_path: Path object for new config file. Should not exist yet - :param selecions: Dict containing selections taken by the user. - """ - from jinja2.exceptions import TemplateNotFound - try: - selections['exchange'] = render_template( - templatefile=f"subtemplates/exchange_{selections['exchange_name']}.j2", - arguments=selections - ) - except TemplateNotFound: - selections['exchange'] = render_template( - templatefile=f"subtemplates/exchange_generic.j2", - arguments=selections - ) - - config_text = render_template(templatefile='base_config.json.j2', - arguments=selections) - - logger.info(f"Writing config to `{config_path}`.") - config_path.write_text(config_text) - - -def start_new_config(args: Dict[str, Any]) -> None: - """ - Create a new strategy from a template - Asking the user questions to fill out the templateaccordingly. - """ - selections = ask_user_config() - config_path = Path(args['config'][0]) - deploy_new_config(config_path, selections) diff --git a/freqtrade/templates/subtemplates/exchange_generic.j2 b/freqtrade/templates/subtemplates/exchange_generic.j2 index 5d5bee2b2..33309de3b 100644 --- a/freqtrade/templates/subtemplates/exchange_generic.j2 +++ b/freqtrade/templates/subtemplates/exchange_generic.j2 @@ -3,7 +3,7 @@ "secret": "{{ exchange_secret }}", "ccxt_config": {"enableRateLimit": true}, "ccxt_async_config": { - "enableRateLimit": true, + "enableRateLimit": true }, "pair_whitelist": [ diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 19999f319..51b69449d 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -557,7 +557,7 @@ def test_start_new_config(mocker, caplog, exchange): 'telegram_token': 'asdf1244', 'telegram_chat_id': '1144444', } - mocker.patch('freqtrade.commands.deploy_commands.ask_user_config', + mocker.patch('freqtrade.commands.build_config_commands.ask_user_config', return_value=sample_selections) args = [ "new-config", From 2f0775fa1b6381b163ecd55ba1a0b684a9bbfea8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 21:30:29 +0100 Subject: [PATCH 102/317] Extract build-config tests to new file --- tests/commands/test_build_config.py | 42 +++++++++++++++++++++++++++++ tests/commands/test_commands.py | 40 ++------------------------- 2 files changed, 44 insertions(+), 38 deletions(-) create mode 100644 tests/commands/test_build_config.py diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py new file mode 100644 index 000000000..4114ff489 --- /dev/null +++ b/tests/commands/test_build_config.py @@ -0,0 +1,42 @@ +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from freqtrade.commands import start_new_config +from tests.conftest import get_args, log_has_re + + +@pytest.mark.parametrize('exchange', ['bittrex', 'binance', 'kraken', 'ftx']) +def test_start_new_config(mocker, caplog, exchange): + wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + sample_selections = { + 'max_open_trades': 3, + 'stake_currency': 'USDT', + 'stake_amount': 100, + 'fiat_display_currency': 'EUR', + 'ticker_interval': '15m', + 'dry_run': True, + 'exchange_name': exchange, + 'exchange_key': 'sampleKey', + 'exchange_secret': 'Samplesecret', + 'telegram': False, + 'telegram_token': 'asdf1244', + 'telegram_chat_id': '1144444', + } + mocker.patch('freqtrade.commands.build_config_commands.ask_user_config', + return_value=sample_selections) + args = [ + "new-config", + "--config", + "coolconfig.json" + ] + start_new_config(get_args(args)) + + assert log_has_re("Writing config to .*", caplog) + assert wt_mock.call_count == 1 + result = json.loads(wt_mock.call_args_list[0][0][0]) + assert result['exchange']['name'] == exchange + assert result['ticker_interval'] == '15m' diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 51b69449d..65d7f6eaf 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1,4 +1,3 @@ -import json import re from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -9,9 +8,8 @@ from freqtrade.commands import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, start_list_strategies, start_list_timeframes, - start_new_config, start_new_hyperopt, - start_new_strategy, start_test_pairlist, - start_trading) + start_new_hyperopt, start_new_strategy, + start_test_pairlist, start_trading) from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode @@ -539,40 +537,6 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): start_new_hyperopt(get_args(args)) -@pytest.mark.parametrize('exchange', ['bittrex', 'binance', 'kraken', 'ftx']) -def test_start_new_config(mocker, caplog, exchange): - wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) - mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - sample_selections = { - 'max_open_trades': 3, - 'stake_currency': 'USDT', - 'stake_amount': 100, - 'fiat_display_currency': 'EUR', - 'ticker_interval': '15m', - 'dry_run': True, - 'exchange_name': exchange, - 'exchange_key': 'sampleKey', - 'exchange_secret': 'Samplesecret', - 'telegram': False, - 'telegram_token': 'asdf1244', - 'telegram_chat_id': '1144444', - } - mocker.patch('freqtrade.commands.build_config_commands.ask_user_config', - return_value=sample_selections) - args = [ - "new-config", - "--config", - "coolconfig.json" - ] - start_new_config(get_args(args)) - - assert log_has_re("Writing config to .*", caplog) - assert wt_mock.call_count == 1 - result = json.loads(wt_mock.call_args_list[0][0][0]) - assert result['exchange']['name'] == exchange - assert result['ticker_interval'] == '15m' - - def test_download_data_keyboardInterrupt(mocker, caplog, markets): dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) From acbf13e648425987d19654123d7e9d2ae83e8da7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 21:47:05 +0100 Subject: [PATCH 103/317] Fail gracefully if user interrupted question session --- freqtrade/commands/build_config_commands.py | 24 ++++++--------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 393416f53..a6623c3cd 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -6,13 +6,14 @@ from questionary import Separator, prompt from freqtrade.exchange import available_exchanges from freqtrade.misc import render_template - +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) def ask_user_config() -> Dict[str, Any]: """ Ask user a few questions to build the configuration. + Interactive questions built using https://github.com/tmbo/questionary :returns: Dict with keys to put into template """ questions = [ @@ -106,23 +107,11 @@ def ask_user_config() -> Dict[str, Any]: ] answers = prompt(questions) - print(answers) + if not answers: + # Interrupted questionary sessions return an empty dict. + raise OperationalException("User interrupted interactive questions.") - sample_selections = { - 'max_open_trades': 3, - 'stake_currency': 'USDT', - 'stake_amount': 100, - 'fiat_display_currency': 'EUR', - 'ticker_interval': '15m', - 'dry_run': True, - 'exchange_name': 'binance', - 'exchange_key': 'sampleKey', - 'exchange_secret': 'Samplesecret', - 'telegram': False, - 'telegram_token': 'asdf1244', - 'telegram_chat_id': '1144444', - } - return sample_selections + return answers def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: @@ -156,5 +145,6 @@ def start_new_config(args: Dict[str, Any]) -> None: Asking the user questions to fill out the templateaccordingly. """ selections = ask_user_config() + config_path = Path(args['config'][0]) deploy_new_config(config_path, selections) From cebf99b5d890cd2979d5e68816b43d4c999bda7e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 21:59:24 +0100 Subject: [PATCH 104/317] Implement validation --- freqtrade/commands/build_config_commands.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index a6623c3cd..6abacd826 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -4,12 +4,29 @@ from typing import Any, Dict from questionary import Separator, prompt +from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.exchange import available_exchanges from freqtrade.misc import render_template from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) +def validate_is_int(val): + try: + _ = int(val) + return True + except Exception: + return False + + +def validate_is_float(val): + try: + _ = float(val) + return True + except Exception: + return False + + def ask_user_config() -> Dict[str, Any]: """ Ask user a few questions to build the configuration. @@ -34,12 +51,14 @@ def ask_user_config() -> Dict[str, Any]: "name": "stake_amount", "message": "Please insert your stake amount:", "default": "0.01", + "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val), }, { "type": "text", "name": "max_open_trades", - "message": "Please insert max_open_trades:", + "message": f"Please insert max_open_trades (Integer or '{UNLIMITED_STAKE_AMOUNT}'):", "default": "3", + "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val) }, { "type": "text", From 83baa6ee2e2c59a4b97ecf76d091cca2c7389154 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Jan 2020 22:47:15 +0100 Subject: [PATCH 105/317] Add test stub --- tests/commands/test_build_config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index 4114ff489..46e79b357 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -4,7 +4,8 @@ from unittest.mock import MagicMock import pytest -from freqtrade.commands import start_new_config +from freqtrade.commands.build_config_commands import (ask_user_config, + start_new_config) from tests.conftest import get_args, log_has_re @@ -40,3 +41,9 @@ def test_start_new_config(mocker, caplog, exchange): result = json.loads(wt_mock.call_args_list[0][0][0]) assert result['exchange']['name'] == exchange assert result['ticker_interval'] == '15m' + + +def test_ask_user_config(): + # TODO: Implement me + pass + # assert ask_user_config() From 4be3f053ca38fc5534a445ac08ff0766708f17df Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Jan 2020 21:42:48 +0100 Subject: [PATCH 106/317] Exclude trading against BNB bases on binance --- .../templates/subtemplates/exchange_binance.j2 | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index 082af45c4..c527d296b 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -24,5 +24,16 @@ "XTZ/BTC" ], "pair_blacklist": [ - "BNB/BTC" + "BNB/BTC", + "BNB/BUSD", + "BNB/ETH", + "BNB/EUR", + "BNB/NGN", + "BNB/PAX", + "BNB/RUB", + "BNB/TRY", + "BNB/TUSD", + "BNB/USDC", + "BNB/USDS", + "BNB/USDT", ] From e2b3907df58737dd0a28854b8ce3d51452e032b0 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 31 Jan 2020 04:39:18 +0100 Subject: [PATCH 107/317] more consistent backtesting tables and labels --- freqtrade/optimize/backtesting.py | 17 ++++++---- freqtrade/optimize/optimize_reports.py | 45 +++++++++++++++++++++---- tests/optimize/test_optimize_reports.py | 32 +++++++++++------- 3 files changed, 68 insertions(+), 26 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cdf74f65f..e2ad0f090 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -404,12 +404,12 @@ class Backtesting: ) # Execute backtest and print results all_results[self.strategy.get_strategy_name()] = self.backtest( - processed=preprocessed, - stake_amount=self.config['stake_amount'], - start_date=min_date, - end_date=max_date, - max_open_trades=max_open_trades, - position_stacking=position_stacking, + processed=preprocessed, + stake_amount=self.config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=max_open_trades, + position_stacking=position_stacking, ) for strategy, results in all_results.items(): @@ -426,7 +426,10 @@ class Backtesting: results=results)) print(' SELL REASON STATS '.center(133, '=')) - print(generate_text_table_sell_reason(data, results)) + print(generate_text_table_sell_reason(data, + stake_currency=self.config['stake_currency'], + max_open_trades=self.config['max_open_trades'], + results=results)) print(' LEFT OPEN TRADES REPORT '.center(133, '=')) print(generate_text_table(data, diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 67056eaa9..6af04d4f2 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -19,9 +19,17 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f') tabular_data = [] - headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', - f'tot profit {stake_currency}', 'tot profit %', 'avg duration', - 'profit', 'loss'] + headers = [ + 'Pair', + 'Buy Count', + 'Avg Profit %', + 'Cum Profit %', + f'Tot Profit {stake_currency}', + 'Tot Profit %', + 'Avg Duration', + 'Wins', + 'Losses' + ] for pair in data: result = results[results.pair == pair] if skip_nan and result.profit_abs.isnull().all(): @@ -58,7 +66,9 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra floatfmt=floatfmt, tablefmt="pipe") # type: ignore -def generate_text_table_sell_reason(data: Dict[str, Dict], results: DataFrame) -> str: +def generate_text_table_sell_reason( + data: Dict[str, Dict], stake_currency: str, max_open_trades: int, results: DataFrame +) -> str: """ Generate small table outlining Backtest results :param data: Dict of containing data that was used during backtesting. @@ -66,13 +76,36 @@ def generate_text_table_sell_reason(data: Dict[str, Dict], results: DataFrame) - :return: pretty printed table with tabulate as string """ tabular_data = [] - headers = ['Sell Reason', 'Count', 'Profit', 'Loss', 'Profit %'] + headers = [ + "Sell Reason", + "Sell Count", + "Wins", + "Losses", + "Avg Profit %", + "Cum Profit %", + f"Tot Profit {stake_currency}", + "Tot Profit %", + ] for reason, count in results['sell_reason'].value_counts().iteritems(): result = results.loc[results['sell_reason'] == reason] profit = len(result[result['profit_abs'] >= 0]) loss = len(result[result['profit_abs'] < 0]) profit_mean = round(result['profit_percent'].mean() * 100.0, 2) - tabular_data.append([reason.value, count, profit, loss, profit_mean]) + profit_sum = round(result["profit_percent"].sum() * 100.0, 2) + profit_tot = result["profit_abs"].sum() + profit_percent_tot = result["profit_percent"].sum() * 100.0 / max_open_trades + tabular_data.append( + [ + reason.value, + count, + profit, + loss, + profit_mean, + profit_sum, + profit_tot, + profit_percent_tot, + ] + ) return tabulate(tabular_data, headers=headers, tablefmt="pipe") diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 518b50d0f..8c1a3619d 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -21,14 +21,14 @@ def test_generate_text_table(default_conf, mocker): ) result_str = ( - '| pair | buy count | avg profit % | cum profit % | ' - 'tot profit BTC | tot profit % | avg duration | profit | loss |\n' - '|:--------|------------:|---------------:|---------------:|' - '-----------------:|---------------:|:---------------|---------:|-------:|\n' - '| ETH/BTC | 2 | 15.00 | 30.00 | ' - '0.60000000 | 15.00 | 0:20:00 | 2 | 0 |\n' - '| TOTAL | 2 | 15.00 | 30.00 | ' - '0.60000000 | 15.00 | 0:20:00 | 2 | 0 |' + '| Pair | Buy Count | Avg Profit % | Cum Profit % | Tot Profit BTC ' + '| Tot Profit % | Avg Duration | Wins | Losses |\n' + '|:--------|------------:|---------------:|---------------:|-----------------:' + '|---------------:|:---------------|-------:|---------:|\n' + '| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 ' + '| 15.00 | 0:20:00 | 2 | 0 |\n' + '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 ' + '| 15.00 | 0:20:00 | 2 | 0 |' ) assert generate_text_table(data={'ETH/BTC': {}}, stake_currency='BTC', max_open_trades=2, @@ -50,13 +50,19 @@ def test_generate_text_table_sell_reason(default_conf, mocker): ) result_str = ( - '| Sell Reason | Count | Profit | Loss | Profit % |\n' - '|:--------------|--------:|---------:|-------:|-----------:|\n' - '| roi | 2 | 2 | 0 | 15 |\n' - '| stop_loss | 1 | 0 | 1 | -10 |' + '| Sell Reason | Sell Count | Wins | Losses | Avg Profit % |' + ' Cum Profit % | Tot Profit BTC | Tot Profit % |\n' + '|:--------------|-------------:|-------:|---------:|---------------:|' + '---------------:|-----------------:|---------------:|\n' + '| roi | 2 | 2 | 0 | 15 |' + ' 30 | 0.6 | 15 |\n' + '| stop_loss | 1 | 0 | 1 | -10 |' + ' -10 | -0.2 | -5 |' ) assert generate_text_table_sell_reason( - data={'ETH/BTC': {}}, results=results) == result_str + data={'ETH/BTC': {}}, + stake_currency='BTC', max_open_trades=2, + results=results) == result_str def test_generate_text_table_strategy(default_conf, mocker): From 907a61152c7be4c1c25c250a7c70bd19e7dae286 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 31 Jan 2020 04:53:37 +0100 Subject: [PATCH 108/317] added rounding to Tot Profit % on Sell Reasosn table to be consistent with other percentiles on table. --- freqtrade/optimize/optimize_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 6af04d4f2..1c558a77c 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -93,7 +93,7 @@ def generate_text_table_sell_reason( profit_mean = round(result['profit_percent'].mean() * 100.0, 2) profit_sum = round(result["profit_percent"].sum() * 100.0, 2) profit_tot = result["profit_abs"].sum() - profit_percent_tot = result["profit_percent"].sum() * 100.0 / max_open_trades + profit_percent_tot = round(result["profit_percent"].sum() * 100.0 / max_open_trades, 2) tabular_data.append( [ reason.value, From c396ad4daa5c582141a3088a3d472e083ec9d094 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 31 Jan 2020 20:41:51 +0100 Subject: [PATCH 109/317] Align quotes in same area --- freqtrade/optimize/optimize_reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 1c558a77c..c5cd944a1 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -92,8 +92,8 @@ def generate_text_table_sell_reason( loss = len(result[result['profit_abs'] < 0]) profit_mean = round(result['profit_percent'].mean() * 100.0, 2) profit_sum = round(result["profit_percent"].sum() * 100.0, 2) - profit_tot = result["profit_abs"].sum() - profit_percent_tot = round(result["profit_percent"].sum() * 100.0 / max_open_trades, 2) + profit_tot = result['profit_abs'].sum() + profit_percent_tot = round(result['profit_percent'].sum() * 100.0 / max_open_trades, 2) tabular_data.append( [ reason.value, From d038bcedb0bd8dee7ae84936a9ff6993d86b4b5b Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 31 Jan 2020 22:37:05 +0100 Subject: [PATCH 110/317] fixed some more line alignments --- freqtrade/optimize/backtesting.py | 12 ++++++------ freqtrade/optimize/hyperopt.py | 29 +++++++++++++++-------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cdf74f65f..7684c5c90 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -404,12 +404,12 @@ class Backtesting: ) # Execute backtest and print results all_results[self.strategy.get_strategy_name()] = self.backtest( - processed=preprocessed, - stake_amount=self.config['stake_amount'], - start_date=min_date, - end_date=max_date, - max_open_trades=max_open_trades, - position_stacking=position_stacking, + processed=preprocessed, + stake_amount=self.config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=max_open_trades, + position_stacking=position_stacking, ) for strategy, results in all_results.items(): diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 525f491f3..ad8b4f2c8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -59,6 +59,7 @@ class Hyperopt: hyperopt = Hyperopt(config) hyperopt.start() """ + def __init__(self, config: Dict[str, Any]) -> None: self.config = config @@ -90,13 +91,13 @@ class Hyperopt: # Populate functions here (hasattr is slow so should not be run during "regular" operations) if hasattr(self.custom_hyperopt, 'populate_indicators'): self.backtesting.strategy.advise_indicators = \ - self.custom_hyperopt.populate_indicators # type: ignore + self.custom_hyperopt.populate_indicators # type: ignore if hasattr(self.custom_hyperopt, 'populate_buy_trend'): self.backtesting.strategy.advise_buy = \ - self.custom_hyperopt.populate_buy_trend # type: ignore + self.custom_hyperopt.populate_buy_trend # type: ignore if hasattr(self.custom_hyperopt, 'populate_sell_trend'): self.backtesting.strategy.advise_sell = \ - self.custom_hyperopt.populate_sell_trend # type: ignore + self.custom_hyperopt.populate_sell_trend # type: ignore # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): @@ -345,15 +346,15 @@ class Hyperopt: if self.has_space('roi'): self.backtesting.strategy.minimal_roi = \ - self.custom_hyperopt.generate_roi_table(params_dict) + self.custom_hyperopt.generate_roi_table(params_dict) if self.has_space('buy'): self.backtesting.strategy.advise_buy = \ - self.custom_hyperopt.buy_strategy_generator(params_dict) + self.custom_hyperopt.buy_strategy_generator(params_dict) if self.has_space('sell'): self.backtesting.strategy.advise_sell = \ - self.custom_hyperopt.sell_strategy_generator(params_dict) + self.custom_hyperopt.sell_strategy_generator(params_dict) if self.has_space('stoploss'): self.backtesting.strategy.stoploss = params_dict['stoploss'] @@ -372,12 +373,12 @@ class Hyperopt: min_date, max_date = get_timerange(processed) backtesting_results = self.backtesting.backtest( - processed=processed, - stake_amount=self.config['stake_amount'], - start_date=min_date, - end_date=max_date, - max_open_trades=self.max_open_trades, - position_stacking=self.position_stacking, + processed=processed, + stake_amount=self.config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=self.max_open_trades, + position_stacking=self.position_stacking, ) return self._get_results_dict(backtesting_results, min_date, max_date, params_dict, params_details) @@ -469,8 +470,8 @@ class Hyperopt: trials = Hyperopt._read_trials(trials_file) if trials[0].get('is_best') is None: raise OperationalException( - "The file with Hyperopt results is incompatible with this version " - "of Freqtrade and cannot be loaded.") + "The file with Hyperopt results is incompatible with this version " + "of Freqtrade and cannot be loaded.") logger.info(f"Loaded {len(trials)} previous evaluations from disk.") return trials From d69ef4380b273468b20c652fb5e6f3d489fc4dd6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 13:44:04 +0100 Subject: [PATCH 111/317] Add basic documentation for new-config option --- docs/utils.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/utils.md b/docs/utils.md index 18deeac54..df2a1c3c3 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -36,6 +36,24 @@ optional arguments: └── sample_strategy.py ``` +## Create new config + +Creates a new configuration file, asking some questions which are important selections for a configuration. + + +``` +usage: freqtrade new-config [-h] [-c PATH] + +optional arguments: + -h, --help show this help message and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` + to read config from stdin. +``` + +!!! Warning + Only vital questions are asked. Freqtrade offers a lot more configuration possibilities, which are listed in the [Configuration documentation](configuration.md#configuration-parameters) + ## Create new strategy Creates a new strategy from a template similar to SampleStrategy. From c40a4d77f8972c80be8be311a9dfd183a1222a64 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 13:46:58 +0100 Subject: [PATCH 112/317] Use exchange_mapping to determine correct exchange-template --- freqtrade/commands/build_config_commands.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 6abacd826..e0910e5b7 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -5,7 +5,7 @@ from typing import Any, Dict from questionary import Separator, prompt from freqtrade.constants import UNLIMITED_STAKE_AMOUNT -from freqtrade.exchange import available_exchanges +from freqtrade.exchange import available_exchanges, MAP_EXCHANGE_CHILDCLASS from freqtrade.misc import render_template from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) @@ -141,8 +141,11 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: """ from jinja2.exceptions import TemplateNotFound try: + exchange_template = MAP_EXCHANGE_CHILDCLASS.get( + selections['exchange_name'], selections['exchange_name']) + selections['exchange'] = render_template( - templatefile=f"subtemplates/exchange_{selections['exchange_name']}.j2", + templatefile=f"subtemplates/exchange_{exchange_template}.j2", arguments=selections ) except TemplateNotFound: From 54512a66ef187aedcddc048968f822b515fa510d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 13:52:25 +0100 Subject: [PATCH 113/317] Update help-strings for list-utils --- docs/utils.md | 51 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index df2a1c3c3..44cbc35d6 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -197,20 +197,31 @@ All exchanges supported by the ccxt library: _1btcxe, acx, adara, allcoin, anxpr Use the `list-timeframes` subcommand to see the list of ticker intervals (timeframes) available for the exchange. ``` -usage: freqtrade list-timeframes [-h] [--exchange EXCHANGE] [-1] +usage: freqtrade list-timeframes [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [-1] optional arguments: - -h, --help show this help message and exit - --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no - config is provided. - -1, --one-column Print output in one column. + -h, --help show this help message and exit + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. + -1, --one-column Print output in one column. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` + to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. ``` * Example: see the timeframes for the 'binance' exchange, set in the configuration file: ``` -$ freqtrade -c config_binance.json list-timeframes +$ freqtrade list-timeframes -c config_binance.json ... Timeframes available for the exchange `binance`: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M ``` @@ -234,14 +245,16 @@ You can print info about any pair/market with these subcommands - and you can fi These subcommands have same usage and same set of available options: ``` -usage: freqtrade list-markets [-h] [--exchange EXCHANGE] [--print-list] - [--print-json] [-1] [--print-csv] +usage: freqtrade list-markets [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [--exchange EXCHANGE] + [--print-list] [--print-json] [-1] [--print-csv] [--base BASE_CURRENCY [BASE_CURRENCY ...]] [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a] -usage: freqtrade list-pairs [-h] [--exchange EXCHANGE] [--print-list] - [--print-json] [-1] [--print-csv] +usage: freqtrade list-pairs [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [--exchange EXCHANGE] + [--print-list] [--print-json] [-1] [--print-csv] [--base BASE_CURRENCY [BASE_CURRENCY ...]] [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a] @@ -260,6 +273,22 @@ optional arguments: Specify quote currency(-ies). Space-separated list. -a, --all Print all pairs or market symbols. By default only active ones are shown. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + ``` By default, only active pairs/markets are shown. Active pairs/markets are those that can currently be traded @@ -281,7 +310,7 @@ $ freqtrade list-pairs --quote USD --print-json human-readable list with summary: ``` -$ freqtrade -c config_binance.json list-pairs --all --base BTC ETH --quote USDT USD --print-list +$ freqtrade list-pairs -c config_binance.json --all --base BTC ETH --quote USDT USD --print-list ``` * Print all markets on exchange "Kraken", in the tabular format: From 8796ecb2a9565e04c336c6e84ac606413c9c8c4c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 13:54:17 +0100 Subject: [PATCH 114/317] Ad example for new-config with answered questions --- docs/installation.md | 3 ++- docs/utils.md | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index cbe000da4..8d3d2c464 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -31,7 +31,7 @@ Freqtrade provides the Linux/MacOS Easy Installation script to install all depen !!! Note Windows installation is explained [here](#windows). -The easiest way to install and run Freqtrade is to clone the bot GitHub repository and then run the Easy Installation script, if it's available for your platform. +The easiest way to install and run Freqtrade is to clone the bot Github repository and then run the Easy Installation script, if it's available for your platform. !!! Note "Version considerations" When cloning the repository the default working branch has the name `develop`. This branch contains all last features (can be considered as relatively stable, thanks to automated tests). The `master` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable). @@ -47,6 +47,7 @@ cd freqtrade git checkout master # Optional, see (1) ./setup.sh --install ``` + (1) This command switches the cloned repository to the use of the `master` branch. It's not needed if you wish to stay on the `develop` branch. You may later switch between branches at any time with the `git checkout master`/`git checkout develop` commands. ## Easy Installation Script (Linux/MacOS) diff --git a/docs/utils.md b/docs/utils.md index 44cbc35d6..f77d2c428 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -54,6 +54,21 @@ optional arguments: !!! Warning Only vital questions are asked. Freqtrade offers a lot more configuration possibilities, which are listed in the [Configuration documentation](configuration.md#configuration-parameters) +### Create config examples + +``` +$ freqtrade new-config --config config_binance.json + +? Do you want to enable Dry-run (simulated trades)? Yes +? Please insert your stake currency: BTC +? Please insert your stake amount: 0.05 +? Please insert max_open_trades (Integer or 'unlimited'): 5 +? Please insert your ticker interval: 15m +? Please insert your display Currency (for reporting): USD +? Select exchange binance +? Do you want to enable Telegram? No +``` + ## Create new strategy Creates a new strategy from a template similar to SampleStrategy. From 929bbe3058e5682793126781956095207d04e3c0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 14:01:19 +0100 Subject: [PATCH 115/317] Link to docker installation from index.md --- docs/index.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index c88c73619..f0ee831e3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -51,12 +51,15 @@ To run this bot we recommend you a cloud instance with a minimum of: ### Software requirements +- Docker (Recommended) + +Alternatively + - Python 3.6.x - pip (pip3) - git - TA-Lib - virtualenv (Recommended) -- Docker (Recommended) ## Support @@ -67,4 +70,4 @@ Click [here](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODc ## Ready to try? -Begin by reading our installation guide [here](installation). +Begin by reading our installation guide [for docker](docker.md), or for [installation without docker](installation.md). From c224c669784cdf0383fab61380e3b65d8189f95a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 14:06:31 +0100 Subject: [PATCH 116/317] Small edits to install.md --- docs/installation.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 8d3d2c464..054cafe9b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -130,6 +130,17 @@ bash setup.sh -i #### 1. Install TA-Lib +Use the provided ta-lib installation script + +```bash +sudo ./build_helpers/install_ta-lib.sh +``` + +!!! Note + This will use the ta-lib tar.gz included in this repository. + +##### TA-Lib manual installation + Official webpage: https://mrjbq7.github.io/ta-lib/install.html ```bash @@ -185,7 +196,8 @@ python3 -m pip install -e . # Initialize the user_directory freqtrade create-userdir --userdir user_data/ -cp config.json.example config.json +# Create a new configuration file +freqtrade new-config --config config.json ``` > *To edit the config please refer to [Bot Configuration](configuration.md).* From cfa6a3e3d32d5575299bb346367ec394ff8a1a7c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 14:12:21 +0100 Subject: [PATCH 117/317] Don't overwrite files --- freqtrade/commands/build_config_commands.py | 6 +++++- tests/commands/test_build_config.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index e0910e5b7..6ba9cf0ac 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -166,7 +166,11 @@ def start_new_config(args: Dict[str, Any]) -> None: Create a new strategy from a template Asking the user questions to fill out the templateaccordingly. """ - selections = ask_user_config() config_path = Path(args['config'][0]) + if config_path.exists(): + raise OperationalException( + f"Configuration `{config_path}` already exists. " + "Please use another configuration name or delete the existing configuration.") + selections = ask_user_config() deploy_new_config(config_path, selections) diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index 46e79b357..b0a048c15 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -6,6 +6,7 @@ import pytest from freqtrade.commands.build_config_commands import (ask_user_config, start_new_config) +from freqtrade.exceptions import OperationalException from tests.conftest import get_args, log_has_re @@ -43,6 +44,17 @@ def test_start_new_config(mocker, caplog, exchange): assert result['ticker_interval'] == '15m' +def test_start_new_config_exists(mocker, caplog): + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + args = [ + "new-config", + "--config", + "coolconfig.json" + ] + with pytest.raises(OperationalException, match=r"Configuration .* already exists\."): + start_new_config(get_args(args)) + + def test_ask_user_config(): # TODO: Implement me pass From d1a3a2d000f1a2e5600265cc4fcf84287fd0122a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 14:22:40 +0100 Subject: [PATCH 118/317] Add tests for build_config --- freqtrade/commands/build_config_commands.py | 23 ++++++++++++++++++--- tests/commands/test_build_config.py | 12 ++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 6ba9cf0ac..838fd510a 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -27,6 +27,19 @@ def validate_is_float(val): return False +def ask_user_overwrite(config_path: Path) -> bool: + questions = [ + { + "type": "confirm", + "name": "overwrite", + "message": f"File {config_path} already exists. Overwrite?", + "default": False, + }, + ] + answers = prompt(questions) + return answers['overwrite'] + + def ask_user_config() -> Dict[str, Any]: """ Ask user a few questions to build the configuration. @@ -169,8 +182,12 @@ def start_new_config(args: Dict[str, Any]) -> None: config_path = Path(args['config'][0]) if config_path.exists(): - raise OperationalException( - f"Configuration `{config_path}` already exists. " - "Please use another configuration name or delete the existing configuration.") + overwrite = ask_user_overwrite(config_path) + if overwrite: + config_path.unlink() + else: + raise OperationalException( + f"Configuration `{config_path}` already exists. " + "Please use another configuration name or delete the existing configuration.") selections = ask_user_config() deploy_new_config(config_path, selections) diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index b0a048c15..8f71c2098 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -1,8 +1,8 @@ -import json from pathlib import Path from unittest.mock import MagicMock import pytest +import rapidjson from freqtrade.commands.build_config_commands import (ask_user_config, start_new_config) @@ -13,7 +13,10 @@ from tests.conftest import get_args, log_has_re @pytest.mark.parametrize('exchange', ['bittrex', 'binance', 'kraken', 'ftx']) def test_start_new_config(mocker, caplog, exchange): wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) - mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + unlink_mock = mocker.patch.object(Path, "unlink", MagicMock()) + mocker.patch('freqtrade.commands.build_config_commands.ask_user_overwrite', return_value=True) + sample_selections = { 'max_open_trades': 3, 'stake_currency': 'USDT', @@ -39,13 +42,16 @@ def test_start_new_config(mocker, caplog, exchange): assert log_has_re("Writing config to .*", caplog) assert wt_mock.call_count == 1 - result = json.loads(wt_mock.call_args_list[0][0][0]) + assert unlink_mock.call_count == 1 + result = rapidjson.loads(wt_mock.call_args_list[0][0][0], + parse_mode=rapidjson.PM_COMMENTS | rapidjson.PM_TRAILING_COMMAS) assert result['exchange']['name'] == exchange assert result['ticker_interval'] == '15m' def test_start_new_config_exists(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch('freqtrade.commands.build_config_commands.ask_user_overwrite', return_value=False) args = [ "new-config", "--config", From 12317b1c535a4ca6b82441896000527c4f69d8cb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 14:46:43 +0100 Subject: [PATCH 119/317] Add some rudimentary tests for questions --- tests/commands/test_build_config.py | 59 ++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index 8f71c2098..d4ebe1de2 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -5,11 +5,33 @@ import pytest import rapidjson from freqtrade.commands.build_config_commands import (ask_user_config, - start_new_config) + ask_user_overwrite, + start_new_config, + validate_is_float, + validate_is_int) from freqtrade.exceptions import OperationalException from tests.conftest import get_args, log_has_re +def test_validate_is_float(): + assert validate_is_float('2.0') + assert validate_is_float('2.1') + assert validate_is_float('0.1') + assert validate_is_float('-0.5') + assert not validate_is_float('-0.5e') + + +def test_validate_is_int(): + assert validate_is_int('2') + assert validate_is_int('6') + assert validate_is_int('-1') + assert validate_is_int('500') + assert not validate_is_int('2.0') + assert not validate_is_int('2.1') + assert not validate_is_int('-2.1') + assert not validate_is_int('-ee') + + @pytest.mark.parametrize('exchange', ['bittrex', 'binance', 'kraken', 'ftx']) def test_start_new_config(mocker, caplog, exchange): wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) @@ -61,7 +83,34 @@ def test_start_new_config_exists(mocker, caplog): start_new_config(get_args(args)) -def test_ask_user_config(): - # TODO: Implement me - pass - # assert ask_user_config() +def test_ask_user_overwrite(mocker): + """ + Once https://github.com/tmbo/questionary/issues/35 is implemented, improve this test. + """ + prompt_mock = mocker.patch('freqtrade.commands.build_config_commands.prompt', + return_value={'overwrite': False}) + assert not ask_user_overwrite(Path('test.json')) + assert prompt_mock.call_count == 1 + + prompt_mock.reset_mock() + prompt_mock = mocker.patch('freqtrade.commands.build_config_commands.prompt', + return_value={'overwrite': True}) + assert ask_user_overwrite(Path('test.json')) + assert prompt_mock.call_count == 1 + + +def test_ask_user_config(mocker): + """ + Once https://github.com/tmbo/questionary/issues/35 is implemented, improve this test. + """ + prompt_mock = mocker.patch('freqtrade.commands.build_config_commands.prompt', + return_value={'overwrite': False}) + answers = ask_user_config() + assert isinstance(answers, dict) + assert prompt_mock.call_count == 1 + + prompt_mock = mocker.patch('freqtrade.commands.build_config_commands.prompt', + return_value={}) + + with pytest.raises(OperationalException, match=r"User interrupted interactive questions\."): + ask_user_config() From 628b06927c08b043338faa11254305bbce948287 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 14:59:14 +0100 Subject: [PATCH 120/317] Support python3.8 virtualenvs and remove config generation via SED --- setup.sh | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/setup.sh b/setup.sh index fb5102e12..bce2e56cf 100755 --- a/setup.sh +++ b/setup.sh @@ -17,6 +17,14 @@ function check_installed_python() { exit 2 fi + which python3.8 + if [ $? -eq 0 ]; then + echo "using Python 3.8" + PYTHON=python3.8 + check_installed_pip + return + fi + which python3.7 if [ $? -eq 0 ]; then echo "using Python 3.7" @@ -215,27 +223,8 @@ function config_generator() { function config() { echo "-------------------------" - echo "Generating config file" + echo "Please use freqtrade new-config -c config.json to generate a new configuration file." echo "-------------------------" - if [ -f config.json ] - then - read -p "A config file already exist, do you want to override it [y/N]? " - if [[ $REPLY =~ ^[Yy]$ ]] - then - config_generator - else - echo "Configuration of config.json ignored." - fi - else - config_generator - fi - - echo - echo "-------------------------" - echo "Config file generated" - echo "-------------------------" - echo "Edit ./config.json to modify Pair and other configurations." - echo } function install() { From 4459679c6404ac1d6aa58c4543eda7e6f4819a19 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 15:14:44 +0100 Subject: [PATCH 121/317] Update dockerfile to 3.8.1 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f631d891d..923285f39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.6-slim-stretch +FROM python:3.8.1-slim-buster RUN apt-get update \ && apt-get -y install curl build-essential libssl-dev \ From 321bc336ea7b63064c41a2f3807fe8838331f56f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 15:14:55 +0100 Subject: [PATCH 122/317] Run tests against 3.8 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53b2e5440..c838baced 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: [ ubuntu-18.04, macos-latest ] - python-version: [3.7] + python-version: [3.7, 3.8] steps: - uses: actions/checkout@v1 From cbd2b265bbb2eea78ad6ffc6b727e1b733729ce5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Feb 2020 15:16:44 +0100 Subject: [PATCH 123/317] Fix small error --- freqtrade/commands/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 504c6b0b5..c8a038328 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -182,7 +182,7 @@ class Arguments: # add new-config subcommand build_config_cmd = subparsers.add_parser('new-config', - help="Create new config") + help="Create new config") build_config_cmd.set_defaults(func=start_new_config) self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd) From f3d500085c0fec96a8ae59bc164e06a55a0beacb Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 2 Feb 2020 07:00:40 +0300 Subject: [PATCH 124/317] Add some type hints --- freqtrade/commands/data_commands.py | 10 +++-- freqtrade/commands/deploy_commands.py | 4 +- freqtrade/commands/plot_commands.py | 2 +- freqtrade/configuration/check_exchange.py | 2 +- .../configuration/deprecated_settings.py | 4 +- .../configuration/directory_operations.py | 2 +- freqtrade/configuration/timerange.py | 5 ++- freqtrade/data/btanalysis.py | 7 +-- freqtrade/data/history.py | 12 ++--- freqtrade/edge/edge_positioning.py | 4 +- freqtrade/exchange/exchange.py | 45 +++++++++++-------- freqtrade/freqtradebot.py | 16 +++---- freqtrade/misc.py | 9 ++-- freqtrade/optimize/backtesting.py | 10 +++-- freqtrade/optimize/hyperopt.py | 18 ++++---- freqtrade/pairlist/IPairList.py | 5 ++- freqtrade/pairlist/PrecisionFilter.py | 6 +-- freqtrade/pairlist/PriceFilter.py | 5 ++- freqtrade/pairlist/VolumePairList.py | 7 +-- freqtrade/persistence.py | 9 ++-- freqtrade/plot/plotting.py | 2 +- freqtrade/resolvers/iresolver.py | 2 +- freqtrade/resolvers/strategy_resolver.py | 7 +-- freqtrade/rpc/rpc.py | 5 ++- freqtrade/rpc/rpc_manager.py | 2 +- freqtrade/strategy/interface.py | 2 +- freqtrade/wallets.py | 10 ++--- freqtrade/worker.py | 2 +- 28 files changed, 114 insertions(+), 100 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index c01772023..aeb598009 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -1,6 +1,6 @@ import logging import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, cast import arrow @@ -43,16 +43,18 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( exchange, pairs=config["pairs"], datadir=config['datadir'], - timerange=timerange, erase=config.get("erase")) + timerange=timerange, erase=cast(bool, config.get("erase"))) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) + datadir=config['datadir'], timerange=timerange, + erase=cast(bool, config.get("erase"))) else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) + datadir=config['datadir'], timerange=timerange, + erase=cast(bool, config.get("erase"))) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 99ae63244..809740661 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -28,7 +28,7 @@ def start_create_userdir(args: Dict[str, Any]) -> None: sys.exit(1) -def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str): +def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: str) -> None: """ Deploy new strategy from template to strategy_path """ @@ -69,7 +69,7 @@ def start_new_strategy(args: Dict[str, Any]) -> None: raise OperationalException("`new-strategy` requires --strategy to be set.") -def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str): +def deploy_new_hyperopt(hyperopt_name: str, hyperopt_path: Path, subtemplate: str) -> None: """ Deploys a new hyperopt template to hyperopt_path """ diff --git a/freqtrade/commands/plot_commands.py b/freqtrade/commands/plot_commands.py index 028933ba7..5e547acb0 100644 --- a/freqtrade/commands/plot_commands.py +++ b/freqtrade/commands/plot_commands.py @@ -5,7 +5,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -def validate_plot_args(args: Dict[str, Any]): +def validate_plot_args(args: Dict[str, Any]) -> None: if not args.get('datadir') and not args.get('config'): raise OperationalException( "You need to specify either `--datadir` or `--config` " diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index 0076b1c5d..92daaf251 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -10,7 +10,7 @@ from freqtrade.state import RunMode logger = logging.getLogger(__name__) -def remove_credentials(config: Dict[str, Any]): +def remove_credentials(config: Dict[str, Any]) -> None: """ Removes exchange keys from the configuration and specifies dry-run Used for backtesting / hyperopt / edge and utils. diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index 78d8218d4..55497d4f5 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) def check_conflicting_settings(config: Dict[str, Any], section1: str, name1: str, - section2: str, name2: str): + section2: str, name2: str) -> None: section1_config = config.get(section1, {}) section2_config = config.get(section2, {}) if name1 in section1_config and name2 in section2_config: @@ -28,7 +28,7 @@ def check_conflicting_settings(config: Dict[str, Any], def process_deprecated_setting(config: Dict[str, Any], section1: str, name1: str, - section2: str, name2: str): + section2: str, name2: str) -> None: section2_config = config.get(section2, {}) if name2 in section2_config: diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py index 43a209483..5f8eb76b0 100644 --- a/freqtrade/configuration/directory_operations.py +++ b/freqtrade/configuration/directory_operations.py @@ -23,7 +23,7 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Pat return folder -def create_userdata_dir(directory: str, create_dir=False) -> Path: +def create_userdata_dir(directory: str, create_dir: bool = False) -> Path: """ Create userdata directory structure. if create_dir is True, then the parent-directory will be created if it does not exist. diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index a8be873df..3db5f6217 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -7,6 +7,7 @@ from typing import Optional import arrow + logger = logging.getLogger(__name__) @@ -30,7 +31,7 @@ class TimeRange: return (self.starttype == other.starttype and self.stoptype == other.stoptype and self.startts == other.startts and self.stopts == other.stopts) - def subtract_start(self, seconds) -> None: + def subtract_start(self, seconds: int) -> None: """ Subtracts from startts if startts is set. :param seconds: Seconds to subtract from starttime @@ -59,7 +60,7 @@ class TimeRange: self.starttype = 'date' @staticmethod - def parse_timerange(text: Optional[str]): + def parse_timerange(text: Optional[str]) -> 'TimeRange': """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 04b2ca980..c28e462ba 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -3,7 +3,7 @@ Helpers when analyzing backtest data """ import logging from pathlib import Path -from typing import Dict +from typing import Dict, Union import numpy as np import pandas as pd @@ -20,7 +20,7 @@ BT_DATA_COLUMNS = ["pair", "profitperc", "open_time", "close_time", "index", "du "open_rate", "close_rate", "open_at_end", "sell_reason"] -def load_backtest_data(filename) -> pd.DataFrame: +def load_backtest_data(filename: Union[Path, str]) -> pd.DataFrame: """ Load backtest data file. :param filename: pathlib.Path object, or string pointing to the file. @@ -151,7 +151,8 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> p return trades -def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], column: str = "close"): +def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], + column: str = "close") -> pd.DataFrame: """ Combine multiple dataframes "column" :param tickers: Dict of Dataframes, dict key should be pair. diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 30d168f78..d891aa5b0 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -86,7 +86,7 @@ def load_tickerdata_file(datadir: Path, pair: str, timeframe: str, def store_tickerdata_file(datadir: Path, pair: str, - timeframe: str, data: list, is_zip: bool = False): + timeframe: str, data: list, is_zip: bool = False) -> None: """ Stores tickerdata to file """ @@ -109,7 +109,7 @@ def load_trades_file(datadir: Path, pair: str, def store_trades_file(datadir: Path, pair: str, - data: list, is_zip: bool = True): + data: list, is_zip: bool = True) -> None: """ Stores tickerdata to file """ @@ -117,7 +117,7 @@ def store_trades_file(datadir: Path, pair: str, misc.file_dump_json(filename, data, is_zip=is_zip) -def _validate_pairdata(pair, pairdata, timerange: TimeRange): +def _validate_pairdata(pair: str, pairdata: List[Dict], timerange: TimeRange) -> None: if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: logger.warning('Missing data at start for pair %s, data starts at %s', pair, arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) @@ -331,7 +331,7 @@ def _download_pair_history(datadir: Path, def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], datadir: Path, timerange: Optional[TimeRange] = None, - erase=False) -> List[str]: + erase: bool = False) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -401,7 +401,7 @@ def _download_trades_history(datadir: Path, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, - timerange: TimeRange, erase=False) -> List[str]: + timerange: TimeRange, erase: bool = False) -> List[str]: """ Refresh stored trades data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -428,7 +428,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], - datadir: Path, timerange: TimeRange, erase=False) -> None: + datadir: Path, timerange: TimeRange, erase: bool = False) -> None: """ Convert stored trades data to ohlcv data """ diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 15883357b..1506b4ed5 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -1,7 +1,7 @@ # pragma pylint: disable=W0603 """ Edge positioning package """ import logging -from typing import Any, Dict, NamedTuple +from typing import Any, Dict, List, NamedTuple import arrow import numpy as np @@ -181,7 +181,7 @@ class Edge: 'strategy stoploss is returned instead.') return self.strategy.stoploss - def adjust(self, pairs) -> list: + def adjust(self, pairs: List[str]) -> list: """ Filters out and sorts "pairs" according to Edge calculated pairs """ diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 87c189457..f7bfb0ee1 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -24,6 +24,12 @@ from freqtrade.exceptions import (DependencyException, InvalidOrderException, from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async from freqtrade.misc import deep_merge_dicts + +# Should probably use typing.Literal when we switch to python 3.8+ +# CcxtModuleType = Literal[ccxt, ccxt_async] +CcxtModuleType = Any + + logger = logging.getLogger(__name__) @@ -51,7 +57,7 @@ class Exchange: } _ft_has: Dict = {} - def __init__(self, config: dict, validate: bool = True) -> None: + def __init__(self, config: Dict[str, Any], validate: bool = True) -> None: """ Initializes this module with the given config, it does basic validation whether the specified exchange and pairs are valid. @@ -135,7 +141,7 @@ class Exchange: if self._api_async and inspect.iscoroutinefunction(self._api_async.close): asyncio.get_event_loop().run_until_complete(self._api_async.close()) - def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt, + def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, ccxt_kwargs: dict = None) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid @@ -224,13 +230,13 @@ class Exchange: markets = self.markets return sorted(set([x['quote'] for _, x in markets.items()])) - def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame: + def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame: if pair_interval in self._klines: return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] else: return DataFrame() - def set_sandbox(self, api, exchange_config: dict, name: str): + def set_sandbox(self, api: ccxt.Exchange, exchange_config: dict, name: str) -> None: if exchange_config.get('sandbox'): if api.urls.get('test'): api.urls['api'] = api.urls['test'] @@ -240,7 +246,7 @@ class Exchange: "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') - def _load_async_markets(self, reload=False) -> None: + def _load_async_markets(self, reload: bool = False) -> None: try: if self._api_async: asyncio.get_event_loop().run_until_complete( @@ -273,7 +279,7 @@ class Exchange: except ccxt.BaseError: logger.exception("Could not reload markets.") - def validate_stakecurrency(self, stake_currency) -> None: + def validate_stakecurrency(self, stake_currency: str) -> None: """ Checks stake-currency against available currencies on the exchange. :param stake_currency: Stake-currency to validate @@ -319,7 +325,7 @@ class Exchange: f"Please check if you are impacted by this restriction " f"on the exchange and eventually remove {pair} from your whitelist.") - def get_valid_pair_combination(self, curr_1, curr_2) -> str: + def get_valid_pair_combination(self, curr_1: str, curr_2: str) -> str: """ Get valid pair combination of curr_1 and curr_2 by trying both combinations. """ @@ -373,7 +379,7 @@ class Exchange: raise OperationalException( f'Time in force policies are not supported for {self.name} yet.') - def validate_required_startup_candles(self, startup_candles) -> None: + def validate_required_startup_candles(self, startup_candles: int) -> None: """ Checks if required startup_candles is more than ohlcv_candle_limit. Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. @@ -392,7 +398,7 @@ class Exchange: """ return endpoint in self._api.has and self._api.has[endpoint] - def amount_to_precision(self, pair, amount: float) -> float: + def amount_to_precision(self, pair: str, amount: float) -> float: ''' Returns the amount to buy or sell to a precision the Exchange accepts Reimplementation of ccxt internal methods - ensuring we can test the result is correct @@ -406,7 +412,7 @@ class Exchange: return amount - def price_to_precision(self, pair, price: float) -> float: + def price_to_precision(self, pair: str, price: float) -> float: ''' Returns the price rounded up to the precision the Exchange accepts. Partial Reimplementation of ccxt internal method decimal_to_precision(), @@ -494,7 +500,7 @@ class Exchange: raise OperationalException(e) from e def buy(self, pair: str, ordertype: str, amount: float, - rate: float, time_in_force) -> Dict: + rate: float, time_in_force: str) -> Dict: if self._config['dry_run']: dry_order = self.dry_run_order(pair, ordertype, "buy", amount, rate) @@ -507,7 +513,7 @@ class Exchange: return self.create_order(pair, ordertype, 'buy', amount, rate, params) def sell(self, pair: str, ordertype: str, amount: float, - rate: float, time_in_force='gtc') -> Dict: + rate: float, time_in_force: str = 'gtc') -> Dict: if self._config['dry_run']: dry_order = self.dry_run_order(pair, ordertype, "sell", amount, rate) @@ -976,8 +982,8 @@ class Exchange: raise OperationalException(e) from e @retrier - def get_fee(self, symbol, type='', side='', amount=1, - price=1, taker_or_maker='maker') -> float: + def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, + price: float = 1, taker_or_maker: str = 'maker') -> float: try: # validate that markets are loaded before trying to get fee if self._api.markets is None or len(self._api.markets) == 0: @@ -1000,7 +1006,7 @@ def get_exchange_bad_reason(exchange_name: str) -> str: return BAD_EXCHANGES.get(exchange_name, "") -def is_exchange_known_ccxt(exchange_name: str, ccxt_module=None) -> bool: +def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool: return exchange_name in ccxt_exchanges(ccxt_module) @@ -1008,14 +1014,14 @@ def is_exchange_officially_supported(exchange_name: str) -> bool: return exchange_name in ['bittrex', 'binance'] -def ccxt_exchanges(ccxt_module=None) -> List[str]: +def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: """ Return the list of all exchanges known to ccxt """ return ccxt_module.exchanges if ccxt_module is not None else ccxt.exchanges -def available_exchanges(ccxt_module=None) -> List[str]: +def available_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: """ Return exchanges available to the bot, i.e. non-bad exchanges in the ccxt list """ @@ -1075,7 +1081,8 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) -def symbol_is_pair(market_symbol: str, base_currency: str = None, quote_currency: str = None): +def symbol_is_pair(market_symbol: str, base_currency: str = None, + quote_currency: str = None) -> bool: """ Check if the market symbol is a pair, i.e. that its symbol consists of the base currency and the quote currency separated by '/' character. If base_currency and/or quote_currency is passed, @@ -1088,7 +1095,7 @@ def symbol_is_pair(market_symbol: str, base_currency: str = None, quote_currency (symbol_parts[1] == quote_currency if quote_currency else len(symbol_parts[1]) > 0)) -def market_is_active(market): +def market_is_active(market: Dict) -> bool: """ Return True if the market is active. """ diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index aac501054..34dbca38e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -265,7 +265,7 @@ class FreqtradeBot: return used_rate - def get_trade_stake_amount(self, pair) -> float: + def get_trade_stake_amount(self, pair: str) -> float: """ Calculate stake amount for the trade :return: float: Stake amount @@ -539,7 +539,7 @@ class FreqtradeBot: return True - def _notify_buy(self, trade: Trade, order_type: str): + def _notify_buy(self, trade: Trade, order_type: str) -> None: """ Sends rpc notification when a buy occured. """ @@ -735,7 +735,7 @@ class FreqtradeBot: return False - def handle_trailing_stoploss_on_exchange(self, trade: Trade, order): + def handle_trailing_stoploss_on_exchange(self, trade: Trade, order: dict) -> None: """ Check to see if stoploss on exchange should be updated in case of trailing stoploss on exchange @@ -758,10 +758,8 @@ class FreqtradeBot: f"for pair {trade.pair}") # Create new stoploss order - if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss, - rate=trade.stop_loss): - return False - else: + if not self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss, + rate=trade.stop_loss): logger.warning(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") @@ -990,7 +988,7 @@ class FreqtradeBot: self._notify_sell(trade, order_type) - def _notify_sell(self, trade: Trade, order_type: str): + def _notify_sell(self, trade: Trade, order_type: str) -> None: """ Sends rpc notification when a sell occured. """ @@ -1031,7 +1029,7 @@ class FreqtradeBot: # Common update trade state methods # - def update_trade_state(self, trade, action_order: dict = None): + def update_trade_state(self, trade: Trade, action_order: dict = None) -> None: """ Checks trades with open orders and updates the amount if necessary """ diff --git a/freqtrade/misc.py b/freqtrade/misc.py index bcba78cf0..2a981c249 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -6,6 +6,7 @@ import logging import re from datetime import datetime from pathlib import Path +from typing import Any from typing.io import IO import numpy as np @@ -40,7 +41,7 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray: return dates.dt.to_pydatetime() -def file_dump_json(filename: Path, data, is_zip=False) -> None: +def file_dump_json(filename: Path, data: Any, is_zip: bool = False) -> None: """ Dump JSON data into a file :param filename: file to create @@ -61,7 +62,7 @@ def file_dump_json(filename: Path, data, is_zip=False) -> None: logger.debug(f'done json to "{filename}"') -def json_load(datafile: IO): +def json_load(datafile: IO) -> Any: """ load data with rapidjson Use this to have a consistent experience, @@ -125,11 +126,11 @@ def round_dict(d, n): return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()} -def plural(num, singular: str, plural: str = None) -> str: +def plural(num: float, singular: str, plural: str = None) -> str: return singular if (num == 1 or num == -1) else plural or singular + 's' -def render_template(templatefile: str, arguments: dict = {}): +def render_template(templatefile: str, arguments: dict = {}) -> str: from jinja2 import Environment, PackageLoader, select_autoescape diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cdf74f65f..ef493e240 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -9,6 +9,7 @@ from datetime import datetime, timedelta from pathlib import Path from typing import Any, Dict, List, NamedTuple, Optional +import arrow from pandas import DataFrame from freqtrade.configuration import (TimeRange, remove_credentials, @@ -24,7 +25,7 @@ from freqtrade.optimize.optimize_reports import ( from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode -from freqtrade.strategy.interface import IStrategy, SellType +from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType logger = logging.getLogger(__name__) @@ -148,7 +149,7 @@ class Backtesting: logger.info(f'Dumping backtest results to {recordfilename}') file_dump_json(recordfilename, records) - def _get_ticker_list(self, processed) -> Dict[str, DataFrame]: + def _get_ticker_list(self, processed: Dict) -> Dict[str, DataFrame]: """ Helper function to convert a processed tickerlist into a list for performance reasons. @@ -175,7 +176,8 @@ class Backtesting: ticker[pair] = [x for x in ticker_data.itertuples()] return ticker - def _get_close_rate(self, sell_row, trade: Trade, sell, trade_dur) -> float: + def _get_close_rate(self, sell_row, trade: Trade, sell: SellCheckTuple, + trade_dur: int) -> float: """ Get close rate for backtesting result """ @@ -280,7 +282,7 @@ class Backtesting: return None def backtest(self, processed: Dict, stake_amount: float, - start_date, end_date, + start_date: arrow.Arrow, end_date: arrow.Arrow, max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame: """ Implement backtesting functionality diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 525f491f3..841f8b6db 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -117,11 +117,11 @@ class Hyperopt: self.print_json = self.config.get('print_json', False) @staticmethod - def get_lock_filename(config) -> str: + def get_lock_filename(config: Dict[str, Any]) -> str: return str(config['user_data_dir'] / 'hyperopt.lock') - def clean_hyperopt(self): + def clean_hyperopt(self) -> None: """ Remove hyperopt pickle files to restart hyperopt. """ @@ -158,7 +158,7 @@ class Hyperopt: f"saved to '{self.trials_file}'.") @staticmethod - def _read_trials(trials_file) -> List: + def _read_trials(trials_file: Path) -> List: """ Read hyperopt trials file """ @@ -189,7 +189,7 @@ class Hyperopt: return result @staticmethod - def print_epoch_details(results, total_epochs, print_json: bool, + def print_epoch_details(results, total_epochs: int, print_json: bool, no_header: bool = False, header_str: str = None) -> None: """ Display details of the hyperopt result @@ -218,7 +218,7 @@ class Hyperopt: Hyperopt._params_pretty_print(params, 'trailing', "Trailing stop:") @staticmethod - def _params_update_for_json(result_dict, params, space: str): + def _params_update_for_json(result_dict, params, space: str) -> None: if space in params: space_params = Hyperopt._space_params(params, space) if space in ['buy', 'sell']: @@ -235,7 +235,7 @@ class Hyperopt: result_dict.update(space_params) @staticmethod - def _params_pretty_print(params, space: str, header: str): + def _params_pretty_print(params, space: str, header: str) -> None: if space in params: space_params = Hyperopt._space_params(params, space, 5) if space == 'stoploss': @@ -251,7 +251,7 @@ class Hyperopt: return round_dict(d, r) if r else d @staticmethod - def is_best_loss(results, current_best_loss) -> bool: + def is_best_loss(results, current_best_loss: float) -> bool: return results['loss'] < current_best_loss def print_results(self, results) -> None: @@ -438,7 +438,7 @@ class Hyperopt: random_state=self.random_state, ) - def fix_optimizer_models_list(self): + def fix_optimizer_models_list(self) -> None: """ WORKAROUND: Since skopt is not actively supported, this resolves problems with skopt memory usage, see also: https://github.com/scikit-optimize/scikit-optimize/pull/746 @@ -460,7 +460,7 @@ class Hyperopt: wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) @staticmethod - def load_previous_results(trials_file) -> List: + def load_previous_results(trials_file: Path) -> List: """ Load data for epochs from the file if we have one """ diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index d722e70f5..1ad4da523 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -7,7 +7,7 @@ Provides lists as configured in config.json import logging from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy -from typing import Dict, List +from typing import Any, Dict, List from freqtrade.exchange import market_is_active @@ -16,7 +16,8 @@ logger = logging.getLogger(__name__) class IPairList(ABC): - def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: """ :param exchange: Exchange instance diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index 5d364795d..f16458ca5 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -48,10 +48,10 @@ class PrecisionFilter(IPairList): """ Filters and sorts pairlists and assigns and returns them again. """ - stoploss = None - if self._config.get('stoploss') is not None: + stoploss = self._config.get('stoploss') + if stoploss is not None: # Precalculate sanitized stoploss value to avoid recalculation for every pair - stoploss = 1 - abs(self._config.get('stoploss')) + stoploss = 1 - abs(stoploss) # Copy list since we're modifying this list for p in deepcopy(pairlist): ticker = tickers.get(p) diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py index b3546ebd9..dc02ae251 100644 --- a/freqtrade/pairlist/PriceFilter.py +++ b/freqtrade/pairlist/PriceFilter.py @@ -1,6 +1,6 @@ import logging from copy import deepcopy -from typing import Dict, List +from typing import Any, Dict, List from freqtrade.pairlist.IPairList import IPairList @@ -9,7 +9,8 @@ logger = logging.getLogger(__name__) class PriceFilter(IPairList): - def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 4ac9935ba..3b28cb7d1 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -6,7 +6,7 @@ Provides lists as configured in config.json """ import logging from datetime import datetime -from typing import Dict, List +from typing import Any, Dict, List from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList @@ -18,7 +18,7 @@ SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] class VolumePairList(IPairList): - def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: dict, pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) @@ -77,7 +77,8 @@ class VolumePairList(IPairList): else: return pairlist - def _gen_pair_whitelist(self, pairlist, tickers, base_currency: str, key: str) -> List[str]: + def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict, + base_currency: str, key: str) -> List[str]: """ Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 75116f1e3..5b0046091 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -64,11 +64,11 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: clean_dry_run_db() -def has_column(columns, searchname: str) -> bool: +def has_column(columns: List, searchname: str) -> bool: return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 -def get_column_def(columns, column: str, default: str) -> str: +def get_column_def(columns: List, column: str, default: str) -> str: return default if not has_column(columns, column) else column @@ -246,14 +246,15 @@ class Trade(_DECL_BASE): if self.initial_stop_loss_pct else None), } - def adjust_min_max_rates(self, current_price: float): + def adjust_min_max_rates(self, current_price: float) -> None: """ Adjust the max_rate and min_rate. """ self.max_rate = max(current_price, self.max_rate or self.open_rate) self.min_rate = min(current_price, self.min_rate or self.open_rate) - def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False): + def adjust_stop_loss(self, current_price: float, stoploss: float, + initial: bool = False) -> None: """ This adjusts the stop loss to it's most recently observed setting :param current_price: Current rate the asset is traded diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 5301d762d..943133ed0 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -370,7 +370,7 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], return fig -def generate_plot_filename(pair, timeframe) -> str: +def generate_plot_filename(pair: str, timeframe: str) -> str: """ Generate filenames per pair/timeframe to be used for storing plots """ diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 5a844097c..3aec5f9e9 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -25,7 +25,7 @@ class IResolver: initial_search_path: Path @classmethod - def build_search_paths(cls, config, user_subdir: Optional[str] = None, + def build_search_paths(cls, config: Dict[str, Any], user_subdir: Optional[str] = None, extra_dir: Optional[str] = None) -> List[Path]: abs_paths: List[Path] = [cls.initial_search_path] diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 9e64f38df..015ba24d9 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -9,7 +9,7 @@ from base64 import urlsafe_b64decode from collections import OrderedDict from inspect import getfullargspec from pathlib import Path -from typing import Dict, Optional +from typing import Any, Dict, Optional from freqtrade.constants import (REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, USERPATH_STRATEGY) @@ -30,7 +30,7 @@ class StrategyResolver(IResolver): initial_search_path = Path(__file__).parent.parent.joinpath('strategy').resolve() @staticmethod - def load_strategy(config: Optional[Dict] = None) -> IStrategy: + def load_strategy(config: Dict[str, Any] = None) -> IStrategy: """ Load the custom class from config parameter :param config: configuration dictionary or None @@ -96,7 +96,8 @@ class StrategyResolver(IResolver): return strategy @staticmethod - def _override_attribute_helper(strategy, config, attribute: str, default): + def _override_attribute_helper(strategy, config: Dict[str, Any], + attribute: str, default: Any): """ Override attributes in the strategy. Prevalence: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 41097c211..7f5cfc101 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -139,7 +139,8 @@ class RPC: results.append(trade_dict) return results - def _rpc_status_table(self, stake_currency, fiat_display_currency: str) -> Tuple[List, List]: + def _rpc_status_table(self, stake_currency: str, + fiat_display_currency: str) -> Tuple[List, List]: trades = Trade.get_open_trades() if not trades: raise RPCException('no active trade') @@ -385,7 +386,7 @@ class RPC: return {'status': 'No more buy will occur from now. Run /reload_conf to reset.'} - def _rpc_forcesell(self, trade_id) -> Dict[str, str]: + def _rpc_forcesell(self, trade_id: str) -> Dict[str, str]: """ Handler for forcesell . Sells the given trade at current price diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index f687fe4d1..670275991 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -61,7 +61,7 @@ class RPCManager: except NotImplementedError: logger.error(f"Message type {msg['type']} not implemented by handler {mod.name}.") - def startup_messages(self, config, pairlist) -> None: + def startup_messages(self, config: Dict[str, Any], pairlist) -> None: if config['dry_run']: self.send_msg({ 'type': RPCMessageType.WARNING_NOTIFICATION, diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 27bc8280e..6e15c5183 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -180,7 +180,7 @@ class IStrategy(ABC): if pair not in self._pair_locked_until or self._pair_locked_until[pair] < until: self._pair_locked_until[pair] = until - def unlock_pair(self, pair) -> None: + def unlock_pair(self, pair: str) -> None: """ Unlocks a pair previously locked using lock_pair. Not used by freqtrade itself, but intended to be used if users lock pairs diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index c52767162..dd5e34fe6 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -30,24 +30,21 @@ class Wallets: self._last_wallet_refresh = 0 self.update() - def get_free(self, currency) -> float: - + def get_free(self, currency: str) -> float: balance = self._wallets.get(currency) if balance and balance.free: return balance.free else: return 0 - def get_used(self, currency) -> float: - + def get_used(self, currency: str) -> float: balance = self._wallets.get(currency) if balance and balance.used: return balance.used else: return 0 - def get_total(self, currency) -> float: - + def get_total(self, currency: str) -> float: balance = self._wallets.get(currency) if balance and balance.total: return balance.total @@ -87,7 +84,6 @@ class Wallets: self._wallets = _wallets def _update_live(self) -> None: - balances = self._exchange.get_balances() for currency in balances: diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 6da04b4a2..64cc97026 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -22,7 +22,7 @@ class Worker: Freqtradebot worker class """ - def __init__(self, args: Dict[str, Any], config=None) -> None: + def __init__(self, args: Dict[str, Any], config: Dict[str, Any] = None) -> None: """ Init all variables and objects the bot needs to work """ From 3499f1b85c883a4ff0298d8d83de158a43d158ec Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sun, 2 Feb 2020 08:47:33 +0100 Subject: [PATCH 125/317] better readability and more consistent with daily sharpe loss method --- freqtrade/optimize/hyperopt_loss_sharpe.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss_sharpe.py index 5631a75de..a4ec6f90a 100644 --- a/freqtrade/optimize/hyperopt_loss_sharpe.py +++ b/freqtrade/optimize/hyperopt_loss_sharpe.py @@ -28,18 +28,19 @@ class SharpeHyperOptLoss(IHyperOptLoss): Uses Sharpe Ratio calculation. """ - total_profit = results.profit_percent + total_profit = results["profit_percent"] days_period = (max_date - min_date).days # adding slippage of 0.1% per trade total_profit = total_profit - 0.0005 - expected_yearly_return = total_profit.sum() / days_period + expected_returns_mean = total_profit.sum() / days_period + up_stdev = np.std(total_profit) if (np.std(total_profit) != 0.): - sharp_ratio = expected_yearly_return / np.std(total_profit) * np.sqrt(365) + sharp_ratio = expected_returns_mean / up_stdev * np.sqrt(365) else: # Define high (negative) sharpe ratio to be clear that this is NOT optimal. sharp_ratio = -20. - # print(expected_yearly_return, np.std(total_profit), sharp_ratio) + # print(expected_returns_mean, up_stdev, sharp_ratio) return -sharp_ratio From d64751687b50e97d3b31d1262f7aeef49fc7aab7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Feb 2020 10:47:44 +0100 Subject: [PATCH 126/317] Fix link and lowercase variable --- docs/exchanges.md | 2 +- freqtrade/exchange/binance.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 18a9f1cba..3c861ce44 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -23,7 +23,7 @@ Binance has been split into 3, and users must use the correct ccxt exchange ID f ## Kraken !!! Tip "Stoploss on Exchange" - Kraken supports `stoploss_on_exchange` and uses stop-loss-market orders. It provides great advantages, so we recommend to benefit from it, however since the resulting order is a stoploss-market order, sell-rates are not guaranteed, which makes this feature less secure than on other exchanges. This limitation is based on kraken's policy [source](https://blog.kraken.com/post/1494/kraken-enables-advanced-orders-and-adds-10-currency-pairs/) and [source2](https://blog.kraken.com/post/1494/kraken-enables-advanced-orders-and-adds-10-currency-pairs/) - which has stoploss-limit orders disabled. + Kraken supports `stoploss_on_exchange` and uses stop-loss-market orders. It provides great advantages, so we recommend to benefit from it, however since the resulting order is a stoploss-market order, sell-rates are not guaranteed, which makes this feature less secure than on other exchanges. This limitation is based on kraken's policy [source](https://blog.kraken.com/post/1234/announcement-delisting-pairs-and-temporary-suspension-of-advanced-order-types/) and [source2](https://blog.kraken.com/post/1494/kraken-enables-advanced-orders-and-adds-10-currency-pairs/) - which has stoploss-limit orders disabled. ### Historic Kraken data diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 45102359d..875628af9 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -46,8 +46,8 @@ class Binance(Exchange): It may work with a limited number of other exchanges, but this has not been tested yet. """ # Limit price threshold: As limit price should always be below stop-price - LIMIT_PRICE_PCT = order_types.get('stoploss_on_exchange_limit_ratio', 0.99) - rate = stop_price * LIMIT_PRICE_PCT + limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99) + rate = stop_price * limit_price_pct ordertype = "stop_loss_limit" From aeabe1800bd2dbe924f454a0a5121bfb81987b9b Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sun, 2 Feb 2020 10:49:00 +0100 Subject: [PATCH 127/317] modified two lines from logger.info to logger.debug cause they're too spammy --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index aac501054..7d13eacd6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -627,7 +627,7 @@ class FreqtradeBot: self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval)) if config_ask_strategy.get('use_order_book', False): - logger.info('Using order book for selling...') + logger.debug(f'Using order book for selling {trade.pair}...') # logger.debug('Order book %s',orderBook) order_book_min = config_ask_strategy.get('order_book_min', 1) order_book_max = config_ask_strategy.get('order_book_max', 1) @@ -636,7 +636,7 @@ class FreqtradeBot: for i in range(order_book_min, order_book_max + 1): order_book_rate = order_book['asks'][i - 1][0] - logger.info(' order book asks top %s: %0.8f', i, order_book_rate) + logger.debug(' order book asks top %s: %0.8f', i, order_book_rate) sell_rate = order_book_rate if self._check_and_execute_sell(trade, sell_rate, buy, sell): From a5e670b4023c2ae50a3441714d607f84ce8b0010 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 2 Feb 2020 18:07:21 +0300 Subject: [PATCH 128/317] Add USERPATH_NOTEBOOKS --- freqtrade/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 53bc4af53..23a60ed0e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -23,6 +23,7 @@ MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons USERPATH_HYPEROPTS = 'hyperopts' USERPATH_STRATEGY = 'strategies' +USERPATH_NOTEBOOKS = 'notebooks' # Soure files with destination directories within user-directory USER_DATA_FILES = { @@ -30,7 +31,7 @@ USER_DATA_FILES = { 'sample_hyperopt_advanced.py': USERPATH_HYPEROPTS, 'sample_hyperopt_loss.py': USERPATH_HYPEROPTS, 'sample_hyperopt.py': USERPATH_HYPEROPTS, - 'strategy_analysis_example.ipynb': 'notebooks', + 'strategy_analysis_example.ipynb': USERPATH_NOTEBOOKS, } SUPPORTED_FIAT = [ From 3fe39a3e1b3e494cd8ebaff1718e809845a324de Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 2 Feb 2020 18:12:23 +0300 Subject: [PATCH 129/317] Rename constant --- freqtrade/commands/deploy_commands.py | 4 ++-- freqtrade/commands/list_commands.py | 4 ++-- freqtrade/constants.py | 4 ++-- freqtrade/resolvers/strategy_resolver.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 99ae63244..e0935f0e5 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -6,7 +6,7 @@ from typing import Any, Dict from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY +from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException from freqtrade.misc import render_template from freqtrade.state import RunMode @@ -57,7 +57,7 @@ def start_new_strategy(args: Dict[str, Any]) -> None: if args["strategy"] == "DefaultStrategy": raise OperationalException("DefaultStrategy is not allowed as name.") - new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py") + new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args["strategy"] + ".py") if new_path.exists(): raise OperationalException(f"`{new_path}` already exists. " diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 022822782..9fe66783d 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -9,7 +9,7 @@ import rapidjson from tabulate import tabulate from freqtrade.configuration import setup_utils_configuration -from freqtrade.constants import USERPATH_STRATEGY +from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException from freqtrade.exchange import (available_exchanges, ccxt_exchanges, market_is_active, symbol_is_pair) @@ -42,7 +42,7 @@ def start_list_strategies(args: Dict[str, Any]) -> None: """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY)) + directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) strategies = StrategyResolver.search_all_objects(directory) # Sort alphabetically strategies = sorted(strategies, key=lambda x: x['name']) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 23a60ed0e..efdd6cc0e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -22,12 +22,12 @@ DRY_RUN_WALLET = 1000 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons USERPATH_HYPEROPTS = 'hyperopts' -USERPATH_STRATEGY = 'strategies' +USERPATH_STRATEGIES = 'strategies' USERPATH_NOTEBOOKS = 'notebooks' # Soure files with destination directories within user-directory USER_DATA_FILES = { - 'sample_strategy.py': USERPATH_STRATEGY, + 'sample_strategy.py': USERPATH_STRATEGIES, 'sample_hyperopt_advanced.py': USERPATH_HYPEROPTS, 'sample_hyperopt_loss.py': USERPATH_HYPEROPTS, 'sample_hyperopt.py': USERPATH_HYPEROPTS, diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 9e64f38df..7f28bd2e6 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import Dict, Optional from freqtrade.constants import (REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, - USERPATH_STRATEGY) + USERPATH_STRATEGIES) from freqtrade.exceptions import OperationalException from freqtrade.resolvers import IResolver from freqtrade.strategy.interface import IStrategy @@ -26,7 +26,7 @@ class StrategyResolver(IResolver): """ object_type = IStrategy object_type_str = "Strategy" - user_subdir = USERPATH_STRATEGY + user_subdir = USERPATH_STRATEGIES initial_search_path = Path(__file__).parent.parent.joinpath('strategy').resolve() @staticmethod @@ -140,7 +140,7 @@ class StrategyResolver(IResolver): """ abs_paths = StrategyResolver.build_search_paths(config, - user_subdir=USERPATH_STRATEGY, + user_subdir=USERPATH_STRATEGIES, extra_dir=extra_dir) if ":" in strategy_name: From 857eb5ff6994cd3a5c8765cf6d43b5b01be1dc3c Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 2 Feb 2020 18:48:29 +0300 Subject: [PATCH 130/317] Add list-hyperopts command --- freqtrade/commands/__init__.py | 1 + freqtrade/commands/arguments.py | 18 +++++++++++++++--- freqtrade/commands/list_commands.py | 22 +++++++++++++++++++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 990c1107a..17723715e 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -14,6 +14,7 @@ from freqtrade.commands.deploy_commands import (start_create_userdir, from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, start_hyperopt_show) from freqtrade.commands.list_commands import (start_list_exchanges, + start_list_hyperopts, start_list_markets, start_list_strategies, start_list_timeframes) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 724814554..0995c89c4 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -32,6 +32,8 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column"] +ARGS_LIST_HYPEROPTS = ["hyperopt_path", "print_one_column"] + ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"] ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"] @@ -132,9 +134,10 @@ class Arguments: from freqtrade.commands import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, - start_list_exchanges, start_list_markets, - start_list_strategies, start_new_hyperopt, - start_new_strategy, start_list_timeframes, + start_list_exchanges, start_list_hyperopts, + start_list_markets, start_list_strategies, + start_list_timeframes, + start_new_hyperopt, start_new_strategy, start_plot_dataframe, start_plot_profit, start_backtesting, start_hyperopt, start_edge, start_test_pairlist, start_trading) @@ -198,6 +201,15 @@ class Arguments: list_strategies_cmd.set_defaults(func=start_list_strategies) self._build_args(optionlist=ARGS_LIST_STRATEGIES, parser=list_strategies_cmd) + # Add list-hyperopts subcommand + list_hyperopts_cmd = subparsers.add_parser( + 'list-hyperopts', + help='Print available hyperopt classes.', + parents=[_common_parser], + ) + list_hyperopts_cmd.set_defaults(func=start_list_hyperopts) + self._build_args(optionlist=ARGS_LIST_HYPEROPTS, parser=list_hyperopts_cmd) + # Add list-exchanges subcommand list_exchanges_cmd = subparsers.add_parser( 'list-exchanges', diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 9fe66783d..f2b6bf995 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -38,7 +38,7 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: def start_list_strategies(args: Dict[str, Any]) -> None: """ - Print Strategies available in a directory + Print files with Strategy custom classes available in the directory """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -54,6 +54,26 @@ def start_list_strategies(args: Dict[str, Any]) -> None: print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) +def start_list_hyperopts(args: Dict[str, Any]) -> None: + """ + Print files with HyperOpt custom classes available in the directory + """ + from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + directory = Path(config.get('hyperopt_path', config['user_data_dir'] / USERPATH_HYPEROPTS)) + hyperopts = HyperOptResolver.search_all_objects(directory) + # Sort alphabetically + hyperopts = sorted(hyperopts, key=lambda x: x['name']) + hyperopts_to_print = [{'name': s['name'], 'location': s['location'].name} for s in hyperopts] + + if args['print_one_column']: + print('\n'.join([s['name'] for s in hyperopts])) + else: + print(tabulate(hyperopts_to_print, headers='keys', tablefmt='pipe')) + + def start_list_timeframes(args: Dict[str, Any]) -> None: """ Print ticker intervals (timeframes) available on Exchange From 505648fb661ea792d555ece1355e241079e9af82 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 2 Feb 2020 18:56:54 +0300 Subject: [PATCH 131/317] Adjust docs --- docs/utils.md | 51 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 18deeac54..b0559f9cc 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -108,9 +108,9 @@ With custom user directory freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt ``` -## List Strategies +## List Strategies and List Hyperopts -Use the `list-strategies` subcommand to see all strategies in one particular directory. +Use the `list-strategies` subcommand to see all strategies in one particular directory and the `list-hyperopts` subcommand to list custom Hyperopts. ``` freqtrade list-strategies --help @@ -133,22 +133,63 @@ Common arguments: --userdir PATH, --user-data-dir PATH Path to userdata directory. ``` +``` +freqtrade list-hyperopts --help +usage: freqtrade list-hyperopts [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [--hyperopt-path PATH] [-1] + +optional arguments: + -h, --help show this help message and exit + --hyperopt-path PATH Specify additional lookup path for Hyperopt and + Hyperopt Loss functions. + -1, --one-column Print output in one column. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` !!! Warning - Using this command will try to load all python files from a directory. This can be a security risk if untrusted files reside in this directory, since all module-level code is executed. + Using these commands will try to load all python files from a directory. This can be a security risk if untrusted files reside in this directory, since all module-level code is executed. -Example: search default strategy directory within userdir +Example: Search default strategies and hyperopts directories (within the default userdir). + +``` bash +freqtrade list-strategies +freqtrade list-hyperopts +``` + +Example: Search strategies and hyperopts directory within the userdir. ``` bash freqtrade list-strategies --userdir ~/.freqtrade/ +freqtrade list-hyperopts --userdir ~/.freqtrade/ ``` -Example: search dedicated strategy path +Example: Search dedicated strategy path. ``` bash freqtrade list-strategies --strategy-path ~/.freqtrade/strategies/ ``` +Example: Search dedicated hyperopt path. + +``` bash +freqtrade list-hyperopt --hyperopt-path ~/.freqtrade/hyperopts/ +``` + ## List Exchanges Use the `list-exchanges` subcommand to see the exchanges available for the bot. From cd0534efcc1a0fa7d748508d22fff7972b643be2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 2 Feb 2020 19:13:17 +0300 Subject: [PATCH 132/317] Add test --- tests/commands/test_commands.py | 36 ++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 65d7f6eaf..c59799190 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -7,7 +7,8 @@ import pytest from freqtrade.commands import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, - start_list_strategies, start_list_timeframes, + start_list_hyperopts, start_list_strategies, + start_list_timeframes, start_new_hyperopt, start_new_strategy, start_test_pairlist, start_trading) from freqtrade.configuration import setup_utils_configuration @@ -665,6 +666,39 @@ def test_start_list_strategies(mocker, caplog, capsys): assert "DefaultStrategy" in captured.out +def test_start_list_hyperopts(mocker, caplog, capsys): + + args = [ + "list-hyperopts", + "--hyperopt-path", + str(Path(__file__).parent.parent / "optimize"), + "-1" + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_hyperopts(pargs) + captured = capsys.readouterr() + assert "TestHyperoptLegacy" not in captured.out + assert "legacy_hyperopt.py" not in captured.out + assert "DefaultHyperOpt" in captured.out + assert "test_hyperopt.py" not in captured.out + + # Test regular output + args = [ + "list-hyperopts", + "--hyperopt-path", + str(Path(__file__).parent.parent / "optimize"), + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_hyperopts(pargs) + captured = capsys.readouterr() + assert "TestHyperoptLegacy" not in captured.out + assert "legacy_hyperopt.py" not in captured.out + assert "DefaultHyperOpt" in captured.out + assert "test_hyperopt.py" in captured.out + + def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): patch_exchange(mocker, mock_markets=True) mocker.patch.multiple('freqtrade.exchange.Exchange', From d12e03e50d30c13f57e5fa661ebf78d388051310 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 2 Feb 2020 20:01:25 +0300 Subject: [PATCH 133/317] Fix test inconsistency in test_freqtradebot.py --- tests/test_freqtradebot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5f16894ab..f334e4eb0 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1145,11 +1145,11 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, # Should not create new order trade.stoploss_order_id = None trade.is_open = False - stoploss_limit.reset_mock() + stoploss.reset_mock() mocker.patch('freqtrade.exchange.Exchange.get_order') - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert stoploss_limit.call_count == 0 + assert stoploss.call_count == 0 def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, From 84156879f6e96e18e6e60589bc2f45d2bb989261 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 2 Feb 2020 20:11:42 +0300 Subject: [PATCH 134/317] Fix NO_CONF_REQUIRED for list-hyperopts --- freqtrade/commands/arguments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 0995c89c4..1931a51be 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -66,8 +66,8 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperopt_show_no_header"] NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs", - "list-strategies", "hyperopt-list", "hyperopt-show", "plot-dataframe", - "plot-profit"] + "list-strategies", "list-hyperopts", "hyperopt-list", "hyperopt-show", + "plot-dataframe", "plot-profit"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] From 2b69e7830d89d9b8ae164511c5b14923ce99080a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Feb 2020 20:02:38 +0100 Subject: [PATCH 135/317] Fix failing CI test --- tests/test_freqtradebot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5f16894ab..f334e4eb0 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1145,11 +1145,11 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, # Should not create new order trade.stoploss_order_id = None trade.is_open = False - stoploss_limit.reset_mock() + stoploss.reset_mock() mocker.patch('freqtrade.exchange.Exchange.get_order') - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert stoploss_limit.call_count == 0 + assert stoploss.call_count == 0 def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, From 537596001e5c4ff030855327095992c48d205724 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 3 Feb 2020 06:20:01 +0300 Subject: [PATCH 136/317] Allow derived strategies --- freqtrade/resolvers/iresolver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 5a844097c..53d08d387 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -61,7 +61,8 @@ class IResolver: valid_objects_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) - if (object_name is None or object_name == name) and cls.object_type in obj.__bases__ + if ((object_name is None or object_name == name) and + issubclass(obj, cls.object_type) and obj is not cls.object_type) ) return valid_objects_gen From c8960ab62893dfcbc15ab6d5cab82ac179908c3d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Feb 2020 06:50:07 +0100 Subject: [PATCH 137/317] Only run coveralls once --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c838baced..8dd61a602 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: pytest --random-order --cov=freqtrade --cov-config=.coveragerc - name: Coveralls - if: startsWith(matrix.os, 'ubuntu') + if: (startsWith(matrix.os, 'ubuntu') && matrix.os == '3.8') env: # Coveralls token. Not used as secret due to github not providing secrets to forked repositories COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu From d0506a643571e74a99a309028403c27bfdf426a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Feb 2020 07:01:07 +0100 Subject: [PATCH 138/317] Use correct matrix variable --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8dd61a602..05d151a88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: pytest --random-order --cov=freqtrade --cov-config=.coveragerc - name: Coveralls - if: (startsWith(matrix.os, 'ubuntu') && matrix.os == '3.8') + if: (startsWith(matrix.os, 'ubuntu') && matrix.python-version == '3.8') env: # Coveralls token. Not used as secret due to github not providing secrets to forked repositories COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu From df249c7c03051ff499e8688e6044990d1100d11c Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Mon, 3 Feb 2020 09:37:50 +0300 Subject: [PATCH 139/317] Remove unclear comment --- freqtrade/exchange/exchange.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index f7bfb0ee1..ede7156a1 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -25,8 +25,6 @@ from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async from freqtrade.misc import deep_merge_dicts -# Should probably use typing.Literal when we switch to python 3.8+ -# CcxtModuleType = Literal[ccxt, ccxt_async] CcxtModuleType = Any From 7b8e6653235d7607bef3e5fb71f08a57bded9571 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2020 08:01:28 +0000 Subject: [PATCH 140/317] Bump jinja2 from 2.10.3 to 2.11.1 Bumps [jinja2](https://github.com/pallets/jinja) from 2.10.3 to 2.11.1. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.10.3...2.11.1) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index e4fe54721..48ba794cb 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -12,7 +12,7 @@ jsonschema==3.2.0 TA-Lib==0.4.17 tabulate==0.8.6 coinmarketcap==5.0.3 -jinja2==2.10.3 +jinja2==2.11.1 # find first, C search in arrays py_find_1st==1.1.4 From bc2ae3e88de22634a3c66b9d9f41349d1ef826c0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2020 08:02:21 +0000 Subject: [PATCH 141/317] Bump pytest from 5.3.4 to 5.3.5 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.3.4 to 5.3.5. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.3.4...5.3.5) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6330d93e5..268c5f777 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ flake8==3.7.9 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.0.0 mypy==0.761 -pytest==5.3.4 +pytest==5.3.5 pytest-asyncio==0.10.0 pytest-cov==2.8.1 pytest-mock==2.0.0 From 401748e9a73757bdf6c04c382b656d901474dfc5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2020 08:02:54 +0000 Subject: [PATCH 142/317] Bump ccxt from 1.21.91 to 1.22.30 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.21.91 to 1.22.30. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.21.91...1.22.30) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index e4fe54721..5f6557161 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.21.91 +ccxt==1.22.30 SQLAlchemy==1.3.13 python-telegram-bot==12.3.0 arrow==0.15.5 From 3938418ad51a7c8dd794b5467a4c61b2e8086c99 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2020 08:03:21 +0000 Subject: [PATCH 143/317] Bump scikit-optimize from 0.5.2 to 0.7.1 Bumps [scikit-optimize](https://github.com/scikit-optimize/scikit-optimize) from 0.5.2 to 0.7.1. - [Release notes](https://github.com/scikit-optimize/scikit-optimize/releases) - [Changelog](https://github.com/scikit-optimize/scikit-optimize/blob/master/CHANGELOG.md) - [Commits](https://github.com/scikit-optimize/scikit-optimize/compare/v0.5.2...v0.7.1) Signed-off-by: dependabot-preview[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 43cad1a0e..202806cef 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -4,6 +4,6 @@ # Required for hyperopt scipy==1.4.1 scikit-learn==0.22.1 -scikit-optimize==0.5.2 +scikit-optimize==0.7.1 filelock==3.0.12 joblib==0.14.1 From d5f704009ffd7fd428201f6432110c344014fd4e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2020 08:03:45 +0000 Subject: [PATCH 144/317] Bump pandas from 0.25.3 to 1.0.0 Bumps [pandas](https://github.com/pandas-dev/pandas) from 0.25.3 to 1.0.0. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v0.25.3...v1.0.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c7dd07ee4..21be02a87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ -r requirements-common.txt numpy==1.18.1 -pandas==0.25.3 +pandas==1.0.0 From f6c09160ab1afda16e0c22acbac7c8b7b608744f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Feb 2020 15:17:36 +0100 Subject: [PATCH 145/317] make sure asyncio_loop is not initialized within ccxt code --- tests/exchange/test_exchange.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 7064d76e1..1121bb035 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -76,9 +76,11 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') caplog.set_level(logging.INFO) conf = copy.deepcopy(default_conf) - conf['exchange']['ccxt_async_config'] = {'aiohttp_trust_env': True} + conf['exchange']['ccxt_async_config'] = {'aiohttp_trust_env': True, 'asyncio_loop': True} ex = Exchange(conf) - assert log_has("Applying additional ccxt config: {'aiohttp_trust_env': True}", caplog) + assert log_has( + "Applying additional ccxt config: {'aiohttp_trust_env': True, 'asyncio_loop': True}", + caplog) assert ex._api_async.aiohttp_trust_env assert not ex._api.aiohttp_trust_env @@ -86,6 +88,8 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog): caplog.clear() conf = copy.deepcopy(default_conf) conf['exchange']['ccxt_config'] = {'TestKWARG': 11} + conf['exchange']['ccxt_async_config'] = {'asyncio_loop': True} + ex = Exchange(conf) assert not log_has("Applying additional ccxt config: {'aiohttp_trust_env': True}", caplog) assert not ex._api_async.aiohttp_trust_env From 684cb54992777d9c7e105033ee1d4d5d918590ee Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 3 Feb 2020 17:17:46 +0300 Subject: [PATCH 146/317] Add pair to exception msg --- freqtrade/exchange/exchange.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a8df4c1bb..c1999b6fa 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -736,10 +736,11 @@ class Exchange: f'Exchange {self._api.name} does not support fetching historical candlestick data.' f'Message: {e}') from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError(f'Could not load ticker history due to {e.__class__.__name__}. ' - f'Message: {e}') from e + raise TemporaryError(f'Could not load ticker history for pair {pair} due to ' + f'{e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch ticker data. Msg: {e}') from e + raise OperationalException(f'Could not fetch ticker data for pair {pair}. ' + f'Msg: {e}') from e @retrier_async async def _async_fetch_trades(self, pair: str, From cbabc295c7c82bb70f7544337f27e9041e982b8b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Feb 2020 20:25:43 +0100 Subject: [PATCH 147/317] Don't convert to datetime - but convert to datetime64 instead --- tests/edge/test_edge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index ef1280fa4..6b86d9c1f 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -163,8 +163,8 @@ def test_edge_results(edge_conf, mocker, caplog, data) -> None: for c, trade in enumerate(data.trades): res = results.iloc[c] assert res.exit_type == trade.sell_reason - assert arrow.get(res.open_time) == _get_frame_time_from_offset(trade.open_tick) - assert arrow.get(res.close_time) == _get_frame_time_from_offset(trade.close_tick) + assert res.open_time == np.datetime64(_get_frame_time_from_offset(trade.open_tick)) + assert res.close_time == np.datetime64(_get_frame_time_from_offset(trade.close_tick)) def test_adjust(mocker, edge_conf): From ffb53a6df5201326fdc65b1b4d15b21ecea2b3ce Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Mon, 3 Feb 2020 23:08:35 +0300 Subject: [PATCH 148/317] get rid of typing.cast() --- freqtrade/commands/data_commands.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index aeb598009..ddc2ca25b 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -1,6 +1,6 @@ import logging import sys -from typing import Any, Dict, List, cast +from typing import Any, Dict, List import arrow @@ -43,18 +43,18 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( exchange, pairs=config["pairs"], datadir=config['datadir'], - timerange=timerange, erase=cast(bool, config.get("erase"))) + timerange=timerange, erase=bool(config.get("erase"))) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( pairs=config["pairs"], timeframes=config["timeframes"], datadir=config['datadir'], timerange=timerange, - erase=cast(bool, config.get("erase"))) + erase=bool(config.get("erase"))) else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=config["pairs"], timeframes=config["timeframes"], datadir=config['datadir'], timerange=timerange, - erase=cast(bool, config.get("erase"))) + erase=bool(config.get("erase"))) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") From 91b4c9668cb019892c79bf47b30d9850a2a32f73 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 4 Feb 2020 01:57:24 +0100 Subject: [PATCH 149/317] More consistency changes... --- freqtrade/freqtradebot.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1a9cbfa64..e51b3d550 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -427,23 +427,23 @@ class FreqtradeBot: Checks depth of market before executing a buy """ conf_bids_to_ask_delta = conf.get('bids_to_ask_delta', 0) - logger.info('checking depth of market for %s', pair) + logger.info(f"Checking depth of market for {pair} ...") order_book = self.exchange.get_order_book(pair, 1000) order_book_data_frame = order_book_to_dataframe(order_book['bids'], order_book['asks']) order_book_bids = order_book_data_frame['b_size'].sum() order_book_asks = order_book_data_frame['a_size'].sum() bids_ask_delta = order_book_bids / order_book_asks logger.info( - f"bids: {order_book_bids}, asks: {order_book_asks}, delta: {bids_ask_delta}, " - f"askprice: {order_book['asks'][0][0]}, bidprice: {order_book['bids'][0][0]}, " - f"immediate ask quantity: {order_book['asks'][0][1]}, " - f"immediate bid quantity: {order_book['bids'][0][1]}", + f"Bids: {order_book_bids}, Asks: {order_book_asks}, Delta: {bids_ask_delta}, " + f"Bid Price: {order_book['bids'][0][0]}, Ask Price: {order_book['asks'][0][0]}, " + f"Immediate Bid Quantity: {order_book['bids'][0][1]}, " + f"Immediate Ask Quantity: {order_book['asks'][0][1]}." ) if bids_ask_delta >= conf_bids_to_ask_delta: - logger.info('bids to ask delta DOES satisfy condition.') + logger.info(f"Bids to asks delta for {pair} DOES satisfy condition.") return True else: - logger.info(f"bids to ask delta for {pair} does not satisfy condition.") + logger.info(f"Bids to asks delta for {pair} does not satisfy condition.") return False def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None) -> bool: From a707aeb3d01b223340ff8ebce02716bfe6777b3d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Feb 2020 07:00:53 +0100 Subject: [PATCH 150/317] Fix implementation of rolling_max --- freqtrade/vendor/qtpylib/indicators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index b3b2ac533..bef140396 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -288,9 +288,9 @@ def rolling_min(series, window=14, min_periods=None): def rolling_max(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: - return series.rolling(window=window, min_periods=min_periods).min() + return series.rolling(window=window, min_periods=min_periods).max() except Exception as e: # noqa: F841 - return pd.Series(series).rolling(window=window, min_periods=min_periods).min() + return pd.Series(series).rolling(window=window, min_periods=min_periods).max() # --------------------------------------------- From aa54fd2251f6144a6c76df33585fa59b4b4886c4 Mon Sep 17 00:00:00 2001 From: untoreh Date: Mon, 3 Feb 2020 07:44:17 +0100 Subject: [PATCH 151/317] - added spread filter - minimum value to volume pairlist --- config_full.json.example | 4 +- docs/configuration.md | 6 +++ freqtrade/constants.py | 3 +- freqtrade/pairlist/SpreadFilter.py | 59 ++++++++++++++++++++++++++++ freqtrade/pairlist/VolumePairList.py | 8 +++- tests/conftest.py | 47 ++++++++++++++++++++++ tests/pairlist/test_pairlist.py | 10 ++++- 7 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 freqtrade/pairlist/SpreadFilter.py diff --git a/config_full.json.example b/config_full.json.example index 82d8bd04a..9f09d2247 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -62,8 +62,8 @@ "refresh_period": 1800 }, {"method": "PrecisionFilter"}, - {"method": "PriceFilter", "low_price_ratio": 0.01 - } + {"method": "PriceFilter", "low_price_ratio": 0.01}, + {"method": "SpreadFilter", "max_spread_ratio": 0.005} ], "exchange": { "name": "bittrex", diff --git a/docs/configuration.md b/docs/configuration.md index fe692eacb..17b9a82c5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -503,6 +503,7 @@ Inactive markets and blacklisted pairs are always removed from the resulting `pa * [`VolumePairList`](#volume-pair-list) * [`PrecisionFilter`](#precision-filter) * [`PriceFilter`](#price-pair-filter) +* [`SpreadFilter`](#spread-filter) !!! Tip "Testing pairlists" Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) subcommand to test your configuration quickly. @@ -551,6 +552,11 @@ Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0. These pairs are dangerous since it may be impossible to place the desired stoploss - and often result in high losses. +#### Spread Filter +Removes pairs that have a difference between asks and bids above the specified ratio (default `0.005`). +Example: +If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027 the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` + ### Full Pairlist example The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting by `quoteVolume` and applies both [`PrecisionFilter`](#precision-filter) and [`PriceFilter`](#price-pair-filter), filtering all assets where 1 priceunit is > 1%. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 53bc4af53..56876e2c9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -17,7 +17,8 @@ REQUIRED_ORDERTIF = ['buy', 'sell'] REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'PriceFilter'] +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', + 'PrecisionFilter', 'PriceFilter', 'SpreadFilter'] DRY_RUN_WALLET = 1000 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons diff --git a/freqtrade/pairlist/SpreadFilter.py b/freqtrade/pairlist/SpreadFilter.py new file mode 100644 index 000000000..9361837cc --- /dev/null +++ b/freqtrade/pairlist/SpreadFilter.py @@ -0,0 +1,59 @@ +import logging +from copy import deepcopy +from typing import Dict, List + +from freqtrade.pairlist.IPairList import IPairList + +logger = logging.getLogger(__name__) + + +class SpreadFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._max_spread_ratio = pairlistconfig.get('max_spread_ratio', 0.005) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return True + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - Filtering pairs with ask/bid diff above " + f"{self._max_spread_ratio * 100}%.") + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ + # Copy list since we're modifying this list + + spread = None + for p in deepcopy(pairlist): + ticker = tickers.get(p) + assert ticker is not None + if 'bid' in ticker and 'ask' in ticker: + spread = 1 - ticker['bid'] / ticker['ask'] + if not ticker or spread > self._max_spread_ratio: + logger.info(f"Removed {ticker['symbol']} from whitelist, " + f"because spread {spread * 100:.3f}% >" + f"{self._max_spread_ratio * 100}%") + pairlist.remove(p) + else: + pairlist.remove(p) + + return pairlist diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 4ac9935ba..3f31f5523 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -28,6 +28,7 @@ class VolumePairList(IPairList): 'for "pairlist.config.number_assets"') self._number_pairs = self._pairlistconfig['number_assets'] self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') + self._min_value = self._pairlistconfig.get('min_value', 0) self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) if not self._exchange.exchange_has('fetchTickers'): @@ -73,11 +74,13 @@ class VolumePairList(IPairList): tickers, self._config['stake_currency'], self._sort_key, + self._min_value ) else: return pairlist - def _gen_pair_whitelist(self, pairlist, tickers, base_currency: str, key: str) -> List[str]: + def _gen_pair_whitelist(self, pairlist, tickers, base_currency: str, + key: str, min_val: int) -> List[str]: """ Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str @@ -96,6 +99,9 @@ class VolumePairList(IPairList): # If other pairlist is in front, use the incomming pairlist. filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + if min_val > 0: + filtered_tickers = list(filter(lambda t: t[key] > min_val, filtered_tickers)) + sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[key]) # Validate whitelist to only have active market pairs diff --git a/tests/conftest.py b/tests/conftest.py index 395388f73..e897dbccd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -640,6 +640,31 @@ def shitcoinmarkets(markets): }, 'info': {}, }, + 'NANO/USDT': { + "percentage": True, + "tierBased": False, + "taker": 0.001, + "maker": 0.001, + "precision": { + "base": 8, + "quote": 8, + "amount": 2, + "price": 4 + }, + "limits": { + }, + "id": "NANOUSDT", + "symbol": "NANO/USDT", + "base": "NANO", + "quote": "USDT", + "baseId": "NANO", + "quoteId": "USDT", + "info": {}, + "type": "spot", + "spot": True, + "future": False, + "active": True + }, }) return shitmarkets @@ -1114,6 +1139,28 @@ def tickers(): 'quoteVolume': 1154.19266394, 'info': {} }, + "NANO/USDT": { + "symbol": "NANO/USDT", + "timestamp": 1580469388244, + "datetime": "2020-01-31T11:16:28.244Z", + "high": 0.7519, + "low": 0.7154, + "bid": 0.7305, + "bidVolume": 300.3, + "ask": 0.7342, + "askVolume": 15.14, + "vwap": 0.73645591, + "open": 0.7154, + "close": 0.7342, + "last": 0.7342, + "previousClose": 0.7189, + "change": 0.0188, + "percentage": 2.628, + "average": None, + "baseVolume": 439472.44, + "quoteVolume": 323652.075405, + "info": {} + }, }) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index ac4cbc813..b8a4be037 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -141,7 +141,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "USDT", ['ETH/USDT']), + "USDT", ['ETH/USDT', 'NANO/USDT']), # No pair for ETH ... ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "ETH", []), @@ -160,6 +160,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.02} ], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + # HOT and XRP are removed because below 1250 quoteVolume + ([{"method": "VolumePairList", "number_assets": 5, + "sort_key": "quoteVolume", "min_value": 1250}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), # StaticPairlist Only ([{"method": "StaticPairList"}, ], "BTC", ['ETH/BTC', 'TKN/BTC']), @@ -167,6 +171,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, ], "BTC", ['TKN/BTC', 'ETH/BTC']), + # SpreadFilter + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "SpreadFilter", "max_spread": 0.005} + ], "USDT", ['ETH/USDT']), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, pairlists, base_currency, whitelist_result, From 6866f6389d45bccf7a261a0f4b3e6362cadb7a0a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Feb 2020 20:41:13 +0100 Subject: [PATCH 152/317] Fix merge-error --- freqtrade/pairlist/VolumePairList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index af6760197..e50dafb63 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -80,7 +80,7 @@ class VolumePairList(IPairList): return pairlist def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict, - base_currency: str, key: str) -> List[str]: + base_currency: str, key: str, min_val: int) -> List[str]: """ Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str From 586cbc750c711ccf8b9bcaa6836be5f8b89d91f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 6 Feb 2020 06:45:11 +0100 Subject: [PATCH 153/317] Add considerations for dry-run --- docs/configuration.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 4ed8bbd0c..72e12f066 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -608,6 +608,14 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo !!! Note A simulated wallet is available during dry-run mode, and will assume a starting capital of `dry_run_wallet` (defaults to 1000). +### Considerations for dry-run + +* API-Keys may or may not be provided. Only Read-Only operations on the exchange are performed in dry-run mode. +* Wallets (`/balance`) will be simulated. +* Orders will be simulated, and will not be posted to the exchange +* In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled. +* Open orders (not Trades!) are reset on bot restart. + ## Switch to production mode In production mode, the bot will engage your money. Be careful, since a wrong From 9639ffb14052f72dd3dc37820c79a8fdf6212bb1 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Thu, 6 Feb 2020 06:49:08 +0100 Subject: [PATCH 154/317] added daily sharpe ratio hyperopt loss method, ty @djacky (#2826) * more consistent backtesting tables and labels * added rounding to Tot Profit % on Sell Reasosn table to be consistent with other percentiles on table. * added daily sharpe ratio hyperopt loss method, ty @djacky * removed commented code * removed unused profit_abs * added proper slippage to each trade * replaced use of old value total_profit * Align quotes in same area * added daily sharpe ratio test and modified hyperopt_loss_sharpe_daily * fixed some more line alignments * updated docs to include SharpeHyperOptLossDaily * Update dockerfile to 3.8.1 * Run tests against 3.8 * added daily sharpe ratio hyperopt loss method, ty @djacky * removed commented code * removed unused profit_abs * added proper slippage to each trade * replaced use of old value total_profit * added daily sharpe ratio test and modified hyperopt_loss_sharpe_daily * updated docs to include SharpeHyperOptLossDaily * docs fixes * missed one fix * fixed standard deviation line * fixed to bracket notation * fixed to bracket notation * fixed syntax error * better readability, kept np.sqrt(365) which results in annualized sharpe ratio * fixed method arguments indentation * updated commented out debug print line * renamed after slippage profit_percent so it wont affect _calculate_results_metrics() * Reworked to fill leading and trailing days * No need for np; make flake happy * Fix risk free rate Co-authored-by: Matthias Co-authored-by: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> --- docs/bot-usage.md | 4 +- docs/hyperopt.md | 38 ++++++------ freqtrade/commands/cli_options.py | 2 +- .../optimize/hyperopt_loss_sharpe_daily.py | 61 +++++++++++++++++++ tests/optimize/test_hyperopt.py | 26 +++++++- 5 files changed, 110 insertions(+), 21 deletions(-) create mode 100644 freqtrade/optimize/hyperopt_loss_sharpe_daily.py diff --git a/docs/bot-usage.md b/docs/bot-usage.md index e856755d2..56e6008a1 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -337,8 +337,8 @@ optional arguments: generate completely different results, since the target for optimization is different. Built-in Hyperopt-loss-functions are: DefaultHyperOptLoss, - OnlyProfitHyperOptLoss, SharpeHyperOptLoss (default: - `DefaultHyperOptLoss`). + OnlyProfitHyperOptLoss, SharpeHyperOptLoss, + SharpeHyperOptLossDaily (default: `DefaultHyperOptLoss`). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/hyperopt.md b/docs/hyperopt.md index f399fe816..3e10f66da 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -57,12 +57,12 @@ Rarely you may also need to override: !!! Tip "Quickly optimize ROI, stoploss and trailing stoploss" You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything (i.e. without creation of a "complete" Hyperopt class with dimensions, parameters, triggers and guards, as described in this document) from the default hyperopt template by relying on your strategy to do most of the calculations. - ``` python + ```python # Have a working strategy at hand. freqtrade new-hyperopt --hyperopt EmptyHyperopt freqtrade hyperopt --hyperopt EmptyHyperopt --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100 - ``` + ``` ### 1. Install a Custom Hyperopt File @@ -75,8 +75,8 @@ Copy the file `user_data/hyperopts/sample_hyperopt.py` into `user_data/hyperopts There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing: -- Inside `indicator_space()` - the parameters hyperopt shall be optimizing. -- Inside `populate_buy_trend()` - applying the parameters. +* Inside `indicator_space()` - the parameters hyperopt shall be optimizing. +* Inside `populate_buy_trend()` - applying the parameters. There you have two different types of indicators: 1. `guards` and 2. `triggers`. @@ -141,7 +141,7 @@ one we call `trigger` and use it to decide which buy trigger we want to use. So let's write the buy strategy using these values: -``` python +```python def populate_buy_trend(dataframe: DataFrame) -> DataFrame: conditions = [] # GUARDS AND TRENDS @@ -192,6 +192,7 @@ Currently, the following loss functions are builtin: * `DefaultHyperOptLoss` (default legacy Freqtrade hyperoptimization loss function) * `OnlyProfitHyperOptLoss` (which takes only amount of profit into consideration) * `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on the trade returns) +* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on daily trade returns) Creation of a custom loss function is covered in the [Advanced Hyperopt](advanced-hyperopt.md) part of the documentation. @@ -206,7 +207,7 @@ We strongly recommend to use `screen` or `tmux` to prevent any connection loss. freqtrade hyperopt --config config.json --hyperopt -e 5000 --spaces all ``` -Use `` as the name of the custom hyperopt used. +Use `` as the name of the custom hyperopt used. The `-e` option will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. @@ -265,23 +266,23 @@ The default Hyperopt Search Space, used when no `--space` command line option is ### Position stacking and disabling max market positions -In some situations, you may need to run Hyperopt (and Backtesting) with the +In some situations, you may need to run Hyperopt (and Backtesting) with the `--eps`/`--enable-position-staking` and `--dmmp`/`--disable-max-market-positions` arguments. By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one -open trade is allowed for every traded pair. The total number of trades open for all pairs +open trade is allowed for every traded pair. The total number of trades open for all pairs is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to some potential trades to be hidden (or masked) by previosly open trades. The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times, -while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades` +while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades` during Hyperopt/Backtesting (which is equal to setting `max_open_trades` to a very high number). !!! Note Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality. -You can also enable position stacking in the configuration file by explicitly setting +You can also enable position stacking in the configuration file by explicitly setting `"position_stacking"=true`. ### Reproducible results @@ -323,7 +324,7 @@ method, what those values match to. So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block: -``` python +```python (dataframe['rsi'] < 29.0) ``` @@ -372,18 +373,19 @@ In order to use this best ROI table found by Hyperopt in backtesting and for liv 118: 0 } ``` + As stated in the comment, you can also use it as the value of the `minimal_roi` setting in the configuration file. #### Default ROI Search Space If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the ticker_interval used. By default the values vary in the following ranges (for some of the most used ticker intervals, values are rounded to 5 digits after the decimal point): -| # step | 1m | | 5m | | 1h | | 1d | | -|---|---|---|---|---|---|---|---|---| -| 1 | 0 | 0.01161...0.11992 | 0 | 0.03...0.31 | 0 | 0.06883...0.71124 | 0 | 0.12178...1.25835 | -| 2 | 2...8 | 0.00774...0.04255 | 10...40 | 0.02...0.11 | 120...480 | 0.04589...0.25238 | 2880...11520 | 0.08118...0.44651 | -| 3 | 4...20 | 0.00387...0.01547 | 20...100 | 0.01...0.04 | 240...1200 | 0.02294...0.09177 | 5760...28800 | 0.04059...0.16237 | -| 4 | 6...44 | 0.0 | 30...220 | 0.0 | 360...2640 | 0.0 | 8640...63360 | 0.0 | +| # step | 1m | | 5m | | 1h | | 1d | | +| ------ | ------ | ----------------- | -------- | ----------- | ---------- | ----------------- | ------------ | ----------------- | +| 1 | 0 | 0.01161...0.11992 | 0 | 0.03...0.31 | 0 | 0.06883...0.71124 | 0 | 0.12178...1.25835 | +| 2 | 2...8 | 0.00774...0.04255 | 10...40 | 0.02...0.11 | 120...480 | 0.04589...0.25238 | 2880...11520 | 0.08118...0.44651 | +| 3 | 4...20 | 0.00387...0.01547 | 20...100 | 0.01...0.04 | 240...1200 | 0.02294...0.09177 | 5760...28800 | 0.04059...0.16237 | +| 4 | 6...44 | 0.0 | 30...220 | 0.0 | 360...2640 | 0.0 | 8640...63360 | 0.0 | These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the ticker interval used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the ticker interval used. @@ -416,6 +418,7 @@ In order to use this best stoploss value found by Hyperopt in backtesting and fo # This attribute will be overridden if the config file contains "stoploss" stoploss = -0.27996 ``` + As stated in the comment, you can also use it as the value of the `stoploss` setting in the configuration file. #### Default Stoploss Search Space @@ -452,6 +455,7 @@ In order to use these best trailing stop parameters found by Hyperopt in backtes trailing_stop_positive_offset = 0.06038 trailing_only_offset_is_reached = True ``` + As stated in the comment, you can also use it as the values of the corresponding settings in the configuration file. #### Default Trailing Stop Search Space diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 490f26cfa..6d8d13129 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -256,7 +256,7 @@ AVAILABLE_CLI_OPTIONS = { help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). ' 'Different functions can generate completely different results, ' 'since the target for optimization is different. Built-in Hyperopt-loss-functions are: ' - 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss.' + 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily.' '(default: `%(default)s`).', metavar='NAME', default=constants.DEFAULT_HYPEROPT_LOSS, diff --git a/freqtrade/optimize/hyperopt_loss_sharpe_daily.py b/freqtrade/optimize/hyperopt_loss_sharpe_daily.py new file mode 100644 index 000000000..d8ea3c5fe --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_sharpe_daily.py @@ -0,0 +1,61 @@ +""" +SharpeHyperOptLossDaily + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +import math +from datetime import datetime + +from pandas import DataFrame, date_range + +from freqtrade.optimize.hyperopt import IHyperOptLoss + + +class SharpeHyperOptLossDaily(IHyperOptLoss): + """ + Defines the loss function for hyperopt. + + This implementation uses the Sharpe Ratio calculation. + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for more optimal results. + + Uses Sharpe Ratio calculation. + """ + resample_freq = '1D' + slippage_per_trade_ratio = 0.0005 + days_in_year = 365 + annual_risk_free_rate = 0.0 + risk_free_rate = annual_risk_free_rate / days_in_year + + # apply slippage per trade to profit_percent + results.loc[:, 'profit_percent_after_slippage'] = \ + results['profit_percent'] - slippage_per_trade_ratio + + # create the index within the min_date and end max_date + t_index = date_range(start=min_date, end=max_date, freq=resample_freq) + + sum_daily = ( + results.resample(resample_freq, on='close_time').agg( + {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0) + ) + + total_profit = sum_daily["profit_percent_after_slippage"] - risk_free_rate + expected_returns_mean = total_profit.mean() + up_stdev = total_profit.std() + + if (up_stdev != 0.): + sharp_ratio = expected_returns_mean / up_stdev * math.sqrt(days_in_year) + else: + # Define high (negative) sharpe ratio to be clear that this is NOT optimal. + sharp_ratio = -20. + + # print(t_index, sum_daily, total_profit) + # print(risk_free_rate, expected_returns_mean, up_stdev, sharp_ratio) + return -sharp_ratio diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 69d110649..b3356bd6d 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -42,7 +42,13 @@ def hyperopt_results(): 'profit_percent': [-0.1, 0.2, 0.3], 'profit_abs': [-0.2, 0.4, 0.6], 'trade_duration': [10, 30, 10], - 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI] + 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI], + 'close_time': + [ + datetime(2019, 1, 1, 9, 26, 3, 478039), + datetime(2019, 2, 1, 9, 26, 3, 478039), + datetime(2019, 3, 1, 9, 26, 3, 478039) + ] } ) @@ -336,6 +342,24 @@ def test_sharpe_loss_prefers_higher_profits(default_conf, hyperopt_results) -> N assert under > correct +def test_sharpe_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: results_over = hyperopt_results.copy() results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 From 5b00eaa42df93ccd583699b98f7d44e43596423c Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Thu, 6 Feb 2020 06:58:58 +0100 Subject: [PATCH 155/317] Updated Strategy Summary table to match other backtesting tables (#2864) --- docs/backtesting.md | 4 ++-- freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/optimize_reports.py | 6 +++--- tests/optimize/test_optimize_reports.py | 12 ++++++------ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 41428085d..2abb32ca0 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -237,8 +237,8 @@ There will be an additional table comparing win/losses of the different strategi Detailed output for all strategies one after the other will be available, so make sure to scroll up to see the details per strategy. ``` -=========================================================== Strategy Summary =========================================================== -| Strategy | buy count | avg profit % | cum profit % | tot profit BTC | tot profit % | avg duration | profit | loss | +=========================================================== STRATEGY SUMMARY =========================================================== +| Strategy | Buy Count | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Losses | |:------------|------------:|---------------:|---------------:|-----------------:|---------------:|:---------------|---------:|-------:| | Strategy1 | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 243 | | Strategy2 | 1487 | -0.13 | -197.58 | -0.00988917 | -98.79 | 4:43:00 | 662 | 825 | diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 96978d407..13c8990a5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -441,7 +441,7 @@ class Backtesting: print() if len(all_results) > 1: # Print Strategy summary table - print(' Strategy Summary '.center(133, '=')) + print(' STRATEGY SUMMARY '.center(133, '=')) print(generate_text_table_strategy(self.config['stake_currency'], self.config['max_open_trades'], all_results=all_results)) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index c5cd944a1..8ad063056 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -121,9 +121,9 @@ def generate_text_table_strategy(stake_currency: str, max_open_trades: str, floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f') tabular_data = [] - headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %', - f'tot profit {stake_currency}', 'tot profit %', 'avg duration', - 'profit', 'loss'] + headers = ['Strategy', 'Buy Count', 'Avg Profit %', 'Cum Profit %', + f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration', + 'Wins', 'Losses'] for strategy, results in all_results.items(): tabular_data.append([ strategy, diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 8c1a3619d..3ea13be47 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -91,14 +91,14 @@ def test_generate_text_table_strategy(default_conf, mocker): ) result_str = ( - '| Strategy | buy count | avg profit % | cum profit % ' - '| tot profit BTC | tot profit % | avg duration | profit | loss |\n' - '|:-----------|------------:|---------------:|---------------:' - '|-----------------:|---------------:|:---------------|---------:|-------:|\n' + '| Strategy | Buy Count | Avg Profit % | Cum Profit % | Tot Profit BTC ' + '| Tot Profit % | Avg Duration | Wins | Losses |\n' + '|:-----------|------------:|---------------:|---------------:|-----------------:' + '|---------------:|:---------------|-------:|---------:|\n' '| ETH/BTC | 3 | 20.00 | 60.00 ' - '| 1.10000000 | 30.00 | 0:17:00 | 3 | 0 |\n' + '| 1.10000000 | 30.00 | 0:17:00 | 3 | 0 |\n' '| LTC/BTC | 3 | 30.00 | 90.00 ' - '| 1.30000000 | 45.00 | 0:20:00 | 3 | 0 |' + '| 1.30000000 | 45.00 | 0:20:00 | 3 | 0 |' ) assert generate_text_table_strategy('BTC', 2, all_results=results) == result_str From 739acaa47502d63dbe24cd89b7dfd7b1babdcbd9 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Thu, 6 Feb 2020 13:54:51 +0300 Subject: [PATCH 156/317] Wordings improved --- docs/configuration.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 72e12f066..c0404d647 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -610,11 +610,11 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo ### Considerations for dry-run -* API-Keys may or may not be provided. Only Read-Only operations on the exchange are performed in dry-run mode. -* Wallets (`/balance`) will be simulated. -* Orders will be simulated, and will not be posted to the exchange +* API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in the dry-run mode. +* Wallets (`/balance`) are simulated. +* Orders are simulated, and will not be posted to the exchange. * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled. -* Open orders (not Trades!) are reset on bot restart. +* Open orders (not trades, which are stored in the database) are reset on bot restart. ## Switch to production mode From 2846f9454fdf48cf90493f91d19f790ddf51cb61 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Thu, 6 Feb 2020 17:02:11 +0300 Subject: [PATCH 157/317] Add description in the docs --- docs/strategy-customization.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index d59b097d7..cc3f8ee33 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -532,6 +532,27 @@ If you want to use a strategy from a different directory you can pass `--strateg freqtrade trade --strategy AwesomeStrategy --strategy-path /some/directory ``` +### Derived strategies + +The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched: + +``` +class MyAwesomeStrategy(IStrategy): + ... + stoploss = 0.13 + trailing_stop = False + # All other attributes and methods are here as they + # should be in any custom strategy... + ... + +class MyAwesomeStrategy2(MyAwesomeStrategy): + # Override something + stoploss = 0.08 + trailing_stop = True +``` + +Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need. The strategy classes may be located in the same module (python file with the source code of your strategy) or in different modules (different python files). In the latter case you need to properly import the strategy class you derive the new one from. + ### Common mistakes when developing strategies Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future. From 412f5d68de2eeefa9b306e5b2604cb695dfc317a Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Thu, 6 Feb 2020 17:42:26 +0300 Subject: [PATCH 158/317] Add description to hyperopt advanced doc chapter --- docs/advanced-hyperopt.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 20af0aaab..f47c33880 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -4,6 +4,34 @@ This page explains some advanced Hyperopt topics that may require higher coding skills and Python knowledge than creation of an ordinal hyperoptimization class. +## Derived hyperopt classes + +Custom hyperop classes can be derived in the same way [it can be done for strategies](strategy-customization.md#derived-strategies). + +Applying to hyperoptimization, as an example, you may override how dimensions are defined in your optimization hyperspace: + +``` +class MyAwesomeHyperOpt(IHyperOpt): + ... + # Uses default stoploss dimension + +class MyAwesomeHyperOpt2(MyAwesomeHyperOpt): + @staticmethod + def stoploss_space() -> List[Dimension]: + # Override boundaries for stoploss + return [ + Real(-0.33, -0.01, name='stoploss'), + ] +``` + +and then quickly switch between hyperopt classes, running optimization process with hyperopt class you need in each particular case: + +``` +$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt ... +or +$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt2 ... +``` + ## Creating and using a custom loss function To use a custom loss function class, make sure that the function `hyperopt_loss_function` is defined in your custom hyperopt loss class. From 2034527faa4c96d518f9a0506799fac89f1f0f56 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Thu, 6 Feb 2020 17:45:15 +0300 Subject: [PATCH 159/317] Update docs/strategy-customization.md Co-Authored-By: Matthias --- docs/strategy-customization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index cc3f8ee33..717baf4db 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -536,7 +536,7 @@ freqtrade trade --strategy AwesomeStrategy --strategy-path /some/directory The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched: -``` +``` python class MyAwesomeStrategy(IStrategy): ... stoploss = 0.13 From 418e7adac159d30e48a21cb09c4547c432fbf67e Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Thu, 6 Feb 2020 17:49:10 +0300 Subject: [PATCH 160/317] Highlight syntax in advanced-hyperopt as well --- docs/advanced-hyperopt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index f47c33880..25b4bd900 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -10,7 +10,7 @@ Custom hyperop classes can be derived in the same way [it can be done for strate Applying to hyperoptimization, as an example, you may override how dimensions are defined in your optimization hyperspace: -``` +```python class MyAwesomeHyperOpt(IHyperOpt): ... # Uses default stoploss dimension From f57bd6b616872904c68dee27fcd3c5814e67d486 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Thu, 6 Feb 2020 21:53:03 +0300 Subject: [PATCH 161/317] Keep the docs clean for unexperienced users --- docs/strategy-customization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 717baf4db..688647c2b 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -551,7 +551,7 @@ class MyAwesomeStrategy2(MyAwesomeStrategy): trailing_stop = True ``` -Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need. The strategy classes may be located in the same module (python file with the source code of your strategy) or in different modules (different python files). In the latter case you need to properly import the strategy class you derive the new one from. +Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need. ### Common mistakes when developing strategies From ff819386e1ce2f03f63acb9601486fcf0280053e Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 7 Feb 2020 03:51:50 +0100 Subject: [PATCH 162/317] added draws to backtesting tables, reduced len of some labels to help fit this without increasing total width --- docs/backtesting.md | 72 ++++++++++++------------- freqtrade/optimize/optimize_reports.py | 25 +++++---- tests/optimize/test_optimize_reports.py | 62 ++++++++++----------- 3 files changed, 79 insertions(+), 80 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 2abb32ca0..79bfa2350 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -119,40 +119,40 @@ A backtesting result will look like that: ``` ========================================================= BACKTESTING REPORT ======================================================== -| pair | buy count | avg profit % | cum profit % | tot profit BTC | tot profit % | avg duration | profit | loss | -|:---------|------------:|---------------:|---------------:|-----------------:|---------------:|:---------------|---------:|-------:| -| ADA/BTC | 35 | -0.11 | -3.88 | -0.00019428 | -1.94 | 4:35:00 | 14 | 21 | -| ARK/BTC | 11 | -0.41 | -4.52 | -0.00022647 | -2.26 | 2:03:00 | 3 | 8 | -| BTS/BTC | 32 | 0.31 | 9.78 | 0.00048938 | 4.89 | 5:05:00 | 18 | 14 | -| DASH/BTC | 13 | -0.08 | -1.07 | -0.00005343 | -0.53 | 4:39:00 | 6 | 7 | -| ENG/BTC | 18 | 1.36 | 24.54 | 0.00122807 | 12.27 | 2:50:00 | 8 | 10 | -| EOS/BTC | 36 | 0.08 | 3.06 | 0.00015304 | 1.53 | 3:34:00 | 16 | 20 | -| ETC/BTC | 26 | 0.37 | 9.51 | 0.00047576 | 4.75 | 6:14:00 | 11 | 15 | -| ETH/BTC | 33 | 0.30 | 9.96 | 0.00049856 | 4.98 | 7:31:00 | 16 | 17 | -| IOTA/BTC | 32 | 0.03 | 1.09 | 0.00005444 | 0.54 | 3:12:00 | 14 | 18 | -| LSK/BTC | 15 | 1.75 | 26.26 | 0.00131413 | 13.13 | 2:58:00 | 6 | 9 | -| LTC/BTC | 32 | -0.04 | -1.38 | -0.00006886 | -0.69 | 4:49:00 | 11 | 21 | -| NANO/BTC | 17 | 1.26 | 21.39 | 0.00107058 | 10.70 | 1:55:00 | 10 | 7 | -| NEO/BTC | 23 | 0.82 | 18.97 | 0.00094936 | 9.48 | 2:59:00 | 10 | 13 | -| REQ/BTC | 9 | 1.17 | 10.54 | 0.00052734 | 5.27 | 3:47:00 | 4 | 5 | -| XLM/BTC | 16 | 1.22 | 19.54 | 0.00097800 | 9.77 | 3:15:00 | 7 | 9 | -| XMR/BTC | 23 | -0.18 | -4.13 | -0.00020696 | -2.07 | 5:30:00 | 12 | 11 | -| XRP/BTC | 35 | 0.66 | 22.96 | 0.00114897 | 11.48 | 3:49:00 | 12 | 23 | -| ZEC/BTC | 22 | -0.46 | -10.18 | -0.00050971 | -5.09 | 2:22:00 | 7 | 15 | -| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 243 | +| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | +|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:| +| ADA/BTC | 35 | -0.11 | -3.88 | -0.00019428 | -1.94 | 4:35:00 | 14 | 0 | 21 | +| ARK/BTC | 11 | -0.41 | -4.52 | -0.00022647 | -2.26 | 2:03:00 | 3 | 0 | 8 | +| BTS/BTC | 32 | 0.31 | 9.78 | 0.00048938 | 4.89 | 5:05:00 | 18 | 0 | 14 | +| DASH/BTC | 13 | -0.08 | -1.07 | -0.00005343 | -0.53 | 4:39:00 | 6 | 0 | 7 | +| ENG/BTC | 18 | 1.36 | 24.54 | 0.00122807 | 12.27 | 2:50:00 | 8 | 0 | 10 | +| EOS/BTC | 36 | 0.08 | 3.06 | 0.00015304 | 1.53 | 3:34:00 | 16 | 0 | 20 | +| ETC/BTC | 26 | 0.37 | 9.51 | 0.00047576 | 4.75 | 6:14:00 | 11 | 0 | 15 | +| ETH/BTC | 33 | 0.30 | 9.96 | 0.00049856 | 4.98 | 7:31:00 | 16 | 0 | 17 | +| IOTA/BTC | 32 | 0.03 | 1.09 | 0.00005444 | 0.54 | 3:12:00 | 14 | 0 | 18 | +| LSK/BTC | 15 | 1.75 | 26.26 | 0.00131413 | 13.13 | 2:58:00 | 6 | 0 | 9 | +| LTC/BTC | 32 | -0.04 | -1.38 | -0.00006886 | -0.69 | 4:49:00 | 11 | 0 | 21 | +| NANO/BTC | 17 | 1.26 | 21.39 | 0.00107058 | 10.70 | 1:55:00 | 10 | 0 | 7 | +| NEO/BTC | 23 | 0.82 | 18.97 | 0.00094936 | 9.48 | 2:59:00 | 10 | 0 | 13 | +| REQ/BTC | 9 | 1.17 | 10.54 | 0.00052734 | 5.27 | 3:47:00 | 4 | 0 | 5 | +| XLM/BTC | 16 | 1.22 | 19.54 | 0.00097800 | 9.77 | 3:15:00 | 7 | 0 | 9 | +| XMR/BTC | 23 | -0.18 | -4.13 | -0.00020696 | -2.07 | 5:30:00 | 12 | 0 | 11 | +| XRP/BTC | 35 | 0.66 | 22.96 | 0.00114897 | 11.48 | 3:49:00 | 12 | 0 | 23 | +| ZEC/BTC | 22 | -0.46 | -10.18 | -0.00050971 | -5.09 | 2:22:00 | 7 | 0 | 15 | +| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 0 | 243 | ========================================================= SELL REASON STATS ========================================================= -| Sell Reason | Count | Profit | Loss | -|:-------------------|--------:|---------:|-------:| -| trailing_stop_loss | 205 | 150 | 55 | -| stop_loss | 166 | 0 | 166 | -| sell_signal | 56 | 36 | 20 | -| force_sell | 2 | 0 | 2 | +| Sell Reason | Sells | Wins | Draws | Losses | +|:-------------------|--------:|------:|-------:|--------:| +| trailing_stop_loss | 205 | 150 | 0 | 55 | +| stop_loss | 166 | 0 | 0 | 166 | +| sell_signal | 56 | 36 | 0 | 20 | +| force_sell | 2 | 0 | 0 | 2 | ====================================================== LEFT OPEN TRADES REPORT ====================================================== -| pair | buy count | avg profit % | cum profit % | tot profit BTC | tot profit % | avg duration | profit | loss | -|:---------|------------:|---------------:|---------------:|-----------------:|---------------:|:---------------|---------:|-------:| -| ADA/BTC | 1 | 0.89 | 0.89 | 0.00004434 | 0.44 | 6:00:00 | 1 | 0 | -| LTC/BTC | 1 | 0.68 | 0.68 | 0.00003421 | 0.34 | 2:00:00 | 1 | 0 | -| TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 | 0 | +| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | +|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:| +| ADA/BTC | 1 | 0.89 | 0.89 | 0.00004434 | 0.44 | 6:00:00 | 1 | 0 | 0 | +| LTC/BTC | 1 | 0.68 | 0.68 | 0.00003421 | 0.34 | 2:00:00 | 1 | 0 | 0 | +| TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 | 0 | 0 | ``` The 1st table contains all trades the bot made, including "left open trades". @@ -238,10 +238,10 @@ Detailed output for all strategies one after the other will be available, so mak ``` =========================================================== STRATEGY SUMMARY =========================================================== -| Strategy | Buy Count | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Losses | -|:------------|------------:|---------------:|---------------:|-----------------:|---------------:|:---------------|---------:|-------:| -| Strategy1 | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 243 | -| Strategy2 | 1487 | -0.13 | -197.58 | -0.00988917 | -98.79 | 4:43:00 | 662 | 825 | +| Strategy | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | +|:------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:| +| Strategy1 | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 0 | 243 | +| Strategy2 | 1487 | -0.13 | -197.58 | -0.00988917 | -98.79 | 4:43:00 | 662 | 0 | 825 | ``` ## Next step diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 8ad063056..b00adbd48 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -21,13 +21,14 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra tabular_data = [] headers = [ 'Pair', - 'Buy Count', + 'Buys', 'Avg Profit %', 'Cum Profit %', f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration', 'Wins', + 'Draws', 'Losses' ] for pair in data: @@ -45,6 +46,7 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra str(timedelta( minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', len(result[result.profit_abs > 0]), + len(result[result.profit_abs == 0]), len(result[result.profit_abs < 0]) ]) @@ -59,6 +61,7 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra str(timedelta( minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', len(results[results.profit_abs > 0]), + len(results[results.profit_abs == 0]), len(results[results.profit_abs < 0]) ]) # Ignore type as floatfmt does allow tuples but mypy does not know that @@ -78,8 +81,9 @@ def generate_text_table_sell_reason( tabular_data = [] headers = [ "Sell Reason", - "Sell Count", + "Sells", "Wins", + "Draws", "Losses", "Avg Profit %", "Cum Profit %", @@ -88,7 +92,8 @@ def generate_text_table_sell_reason( ] for reason, count in results['sell_reason'].value_counts().iteritems(): result = results.loc[results['sell_reason'] == reason] - profit = len(result[result['profit_abs'] >= 0]) + wins = len(result[result['profit_abs'] > 0]) + draws = len(result[result['profit_abs'] == 0]) loss = len(result[result['profit_abs'] < 0]) profit_mean = round(result['profit_percent'].mean() * 100.0, 2) profit_sum = round(result["profit_percent"].sum() * 100.0, 2) @@ -98,7 +103,8 @@ def generate_text_table_sell_reason( [ reason.value, count, - profit, + wins, + draws, loss, profit_mean, profit_sum, @@ -121,9 +127,9 @@ def generate_text_table_strategy(stake_currency: str, max_open_trades: str, floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f') tabular_data = [] - headers = ['Strategy', 'Buy Count', 'Avg Profit %', 'Cum Profit %', + headers = ['Strategy', 'Buys', 'Avg Profit %', 'Cum Profit %', f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration', - 'Wins', 'Losses'] + 'Wins', 'Draws', 'Losses'] for strategy, results in all_results.items(): tabular_data.append([ strategy, @@ -135,6 +141,7 @@ def generate_text_table_strategy(stake_currency: str, max_open_trades: str, str(timedelta( minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', len(results[results.profit_abs > 0]), + len(results[results.profit_abs == 0]), len(results[results.profit_abs < 0]) ]) # Ignore type as floatfmt does allow tuples but mypy does not know that @@ -146,9 +153,9 @@ def generate_edge_table(results: dict) -> str: floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', '.d') tabular_data = [] - headers = ['pair', 'stoploss', 'win rate', 'risk reward ratio', - 'required risk reward', 'expectancy', 'total number of trades', - 'average duration (min)'] + headers = ['Pair', 'Stoploss', 'Win Rate', 'Risk Reward Ratio', + 'Required Risk Reward', 'Expectancy', 'Total Number of Trades', + 'Average Duration (min)'] for result in results.items(): if result[1].nb_trades > 0: diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 3ea13be47..c31b10290 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -15,20 +15,17 @@ def test_generate_text_table(default_conf, mocker): 'profit_percent': [0.1, 0.2], 'profit_abs': [0.2, 0.4], 'trade_duration': [10, 30], - 'profit': [2, 0], - 'loss': [0, 0] + 'wins': [2, 0], + 'draws': [0, 0], + 'losses': [0, 0] } ) result_str = ( - '| Pair | Buy Count | Avg Profit % | Cum Profit % | Tot Profit BTC ' - '| Tot Profit % | Avg Duration | Wins | Losses |\n' - '|:--------|------------:|---------------:|---------------:|-----------------:' - '|---------------:|:---------------|-------:|---------:|\n' - '| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 ' - '| 15.00 | 0:20:00 | 2 | 0 |\n' - '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 ' - '| 15.00 | 0:20:00 | 2 | 0 |' + '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|:--------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|-------:|--------:|---------:|\n' + '| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 | 15.00 | 0:20:00 | 2 | 0 | 0 |\n' + '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 | 15.00 | 0:20:00 | 2 | 0 | 0 |' ) assert generate_text_table(data={'ETH/BTC': {}}, stake_currency='BTC', max_open_trades=2, @@ -43,21 +40,18 @@ def test_generate_text_table_sell_reason(default_conf, mocker): 'profit_percent': [0.1, 0.2, -0.1], 'profit_abs': [0.2, 0.4, -0.2], 'trade_duration': [10, 30, 10], - 'profit': [2, 0, 0], - 'loss': [0, 0, 1], + 'wins': [2, 0, 0], + 'draws': [0, 0, 0], + 'losses': [0, 0, 1], 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] } ) result_str = ( - '| Sell Reason | Sell Count | Wins | Losses | Avg Profit % |' - ' Cum Profit % | Tot Profit BTC | Tot Profit % |\n' - '|:--------------|-------------:|-------:|---------:|---------------:|' - '---------------:|-----------------:|---------------:|\n' - '| roi | 2 | 2 | 0 | 15 |' - ' 30 | 0.6 | 15 |\n' - '| stop_loss | 1 | 0 | 1 | -10 |' - ' -10 | -0.2 | -5 |' + '| Sell Reason | Sells | Wins | Draws | Losses | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % |\n' + '|:--------------|--------:|-------:|--------:|---------:|---------------:|---------------:|-----------------:|---------------:|\n' + '| roi | 2 | 2 | 0 | 0 | 15 | 30 | 0.6 | 15 |\n' + '| stop_loss | 1 | 0 | 0 | 1 | -10 | -10 | -0.2 | -5 |' ) assert generate_text_table_sell_reason( data={'ETH/BTC': {}}, @@ -67,38 +61,36 @@ def test_generate_text_table_sell_reason(default_conf, mocker): def test_generate_text_table_strategy(default_conf, mocker): results = {} - results['ETH/BTC'] = pd.DataFrame( + results['TestStrategy1'] = pd.DataFrame( { 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], 'profit_percent': [0.1, 0.2, 0.3], 'profit_abs': [0.2, 0.4, 0.5], 'trade_duration': [10, 30, 10], - 'profit': [2, 0, 0], - 'loss': [0, 0, 1], + 'wins': [2, 0, 0], + 'draws': [0, 0, 0], + 'losses': [0, 0, 1], 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] } ) - results['LTC/BTC'] = pd.DataFrame( + results['TestStrategy2'] = pd.DataFrame( { 'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'], 'profit_percent': [0.4, 0.2, 0.3], 'profit_abs': [0.4, 0.4, 0.5], 'trade_duration': [15, 30, 15], - 'profit': [4, 1, 0], - 'loss': [0, 0, 1], + 'wins': [4, 1, 0], + 'draws': [0, 0, 0], + 'losses': [0, 0, 1], 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] } ) result_str = ( - '| Strategy | Buy Count | Avg Profit % | Cum Profit % | Tot Profit BTC ' - '| Tot Profit % | Avg Duration | Wins | Losses |\n' - '|:-----------|------------:|---------------:|---------------:|-----------------:' - '|---------------:|:---------------|-------:|---------:|\n' - '| ETH/BTC | 3 | 20.00 | 60.00 ' - '| 1.10000000 | 30.00 | 0:17:00 | 3 | 0 |\n' - '| LTC/BTC | 3 | 30.00 | 90.00 ' - '| 1.30000000 | 45.00 | 0:20:00 | 3 | 0 |' + '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|:--------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|-------:|--------:|---------:|\n' + '| TestStrategy1 | 3 | 20.00 | 60.00 | 1.10000000 | 30.00 | 0:17:00 | 3 | 0 | 0 |\n' + '| TestStrategy2 | 3 | 30.00 | 90.00 | 1.30000000 | 45.00 | 0:20:00 | 3 | 0 | 0 |' ) assert generate_text_table_strategy('BTC', 2, all_results=results) == result_str @@ -111,4 +103,4 @@ def test_generate_edge_table(edge_conf, mocker): assert generate_edge_table(results).count(':|') == 7 assert generate_edge_table(results).count('| ETH/BTC |') == 1 assert generate_edge_table(results).count( - '| risk reward ratio | required risk reward | expectancy |') == 1 + '| Risk Reward Ratio | Required Risk Reward | Expectancy |') == 1 From aa2cb937b145f7a87b8044fe17e4d945c617ba2c Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 7 Feb 2020 03:54:47 +0100 Subject: [PATCH 163/317] flake8 :) --- tests/optimize/test_optimize_reports.py | 36 ++++++++++++++++--------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index c31b10290..57e928cca 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -22,10 +22,14 @@ def test_generate_text_table(default_conf, mocker): ) result_str = ( - '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' - '|:--------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|-------:|--------:|---------:|\n' - '| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 | 15.00 | 0:20:00 | 2 | 0 | 0 |\n' - '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 | 15.00 | 0:20:00 | 2 | 0 | 0 |' + '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' + ' Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|:--------|-------:|---------------:|---------------:|-----------------:|' + '---------------:|:---------------|-------:|--------:|---------:|\n' + '| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 |' + ' 15.00 | 0:20:00 | 2 | 0 | 0 |\n' + '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 |' + ' 15.00 | 0:20:00 | 2 | 0 | 0 |' ) assert generate_text_table(data={'ETH/BTC': {}}, stake_currency='BTC', max_open_trades=2, @@ -48,10 +52,14 @@ def test_generate_text_table_sell_reason(default_conf, mocker): ) result_str = ( - '| Sell Reason | Sells | Wins | Draws | Losses | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % |\n' - '|:--------------|--------:|-------:|--------:|---------:|---------------:|---------------:|-----------------:|---------------:|\n' - '| roi | 2 | 2 | 0 | 0 | 15 | 30 | 0.6 | 15 |\n' - '| stop_loss | 1 | 0 | 0 | 1 | -10 | -10 | -0.2 | -5 |' + '| Sell Reason | Sells | Wins | Draws | Losses |' + ' Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % |\n' + '|:--------------|--------:|-------:|--------:|---------:|' + '---------------:|---------------:|-----------------:|---------------:|\n' + '| roi | 2 | 2 | 0 | 0 |' + ' 15 | 30 | 0.6 | 15 |\n' + '| stop_loss | 1 | 0 | 0 | 1 |' + ' -10 | -10 | -0.2 | -5 |' ) assert generate_text_table_sell_reason( data={'ETH/BTC': {}}, @@ -87,10 +95,14 @@ def test_generate_text_table_strategy(default_conf, mocker): ) result_str = ( - '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' - '|:--------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|-------:|--------:|---------:|\n' - '| TestStrategy1 | 3 | 20.00 | 60.00 | 1.10000000 | 30.00 | 0:17:00 | 3 | 0 | 0 |\n' - '| TestStrategy2 | 3 | 30.00 | 90.00 | 1.30000000 | 45.00 | 0:20:00 | 3 | 0 | 0 |' + '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' + ' Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|:--------------|-------:|---------------:|---------------:|------' + '-----------:|---------------:|:---------------|-------:|--------:|---------:|\n' + '| TestStrategy1 | 3 | 20.00 | 60.00 | ' + ' 1.10000000 | 30.00 | 0:17:00 | 3 | 0 | 0 |\n' + '| TestStrategy2 | 3 | 30.00 | 90.00 | ' + ' 1.30000000 | 45.00 | 0:20:00 | 3 | 0 | 0 |' ) assert generate_text_table_strategy('BTC', 2, all_results=results) == result_str From a893f70e49188ad7711d0b9db4b24780baeb20c9 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 8 Feb 2020 02:21:39 +0300 Subject: [PATCH 164/317] Replace NXT with XRP in config.json.example --- config.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index 8b85e71eb..c62b8240c 100644 --- a/config.json.example +++ b/config.json.example @@ -44,7 +44,7 @@ "DASH/BTC", "ZEC/BTC", "XLM/BTC", - "NXT/BTC", + "XRP/BTC", "TRX/BTC", "ADA/BTC", "XMR/BTC" From 28184201e471385d29a48629ce6a6bac7bbb39ec Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 8 Feb 2020 02:47:50 +0300 Subject: [PATCH 165/317] Align sample_hyperopt_advanced.py to hyperopt_interface.py --- freqtrade/templates/sample_hyperopt_advanced.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/freqtrade/templates/sample_hyperopt_advanced.py b/freqtrade/templates/sample_hyperopt_advanced.py index b4bbee3fb..e66ef948b 100644 --- a/freqtrade/templates/sample_hyperopt_advanced.py +++ b/freqtrade/templates/sample_hyperopt_advanced.py @@ -230,7 +230,7 @@ class AdvancedSampleHyperOpt(IHyperOpt): 'stoploss' optimization hyperspace. """ return [ - Real(-0.5, -0.02, name='stoploss'), + Real(-0.35, -0.02, name='stoploss'), ] @staticmethod @@ -249,8 +249,15 @@ class AdvancedSampleHyperOpt(IHyperOpt): # other 'trailing' hyperspace parameters. Categorical([True], name='trailing_stop'), - Real(0.02, 0.35, name='trailing_stop_positive'), - Real(0.01, 0.1, name='trailing_stop_positive_offset'), + Real(0.01, 0.35, name='trailing_stop_positive'), + + # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive', + # so this intermediate parameter is used as the value of the difference between + # them. The value of the 'trailing_stop_positive_offset' is constructed in the + # generate_trailing_params() method. + # This is similar to the hyperspace dimensions used for constructing the ROI tables. + Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'), + Categorical([True, False], name='trailing_only_offset_is_reached'), ] From 61ced5e926db56740550192db1cba34883a81374 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 8 Feb 2020 02:49:06 +0300 Subject: [PATCH 166/317] Fix typo --- freqtrade/optimize/hyperopt_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index d7d917c19..b3cedef2c 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -207,7 +207,7 @@ class IHyperOpt(ABC): # so this intermediate parameter is used as the value of the difference between # them. The value of the 'trailing_stop_positive_offset' is constructed in the # generate_trailing_params() method. - # # This is similar to the hyperspace dimensions used for constructing the ROI tables. + # This is similar to the hyperspace dimensions used for constructing the ROI tables. Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'), Categorical([True, False], name='trailing_only_offset_is_reached'), From a1fe3850e29d6cd04bbb4e2df182885dcca601c7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Feb 2020 13:34:04 +0100 Subject: [PATCH 167/317] Improve docker-compose file to be ready to use --- docker-compose.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index cae98c3ee..3a4c4c2db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,18 @@ version: '3' services: freqtrade: image: freqtradeorg/freqtrade:master + # Build step - only needed when additional dependencies are needed + # build: + # context: . + # dockerfile: "./Dockerfile.technical" + restart: unless-stopped + container_name: freqtrade volumes: - "./user_data:/freqtrade/user_data" - - "./config.json:/freqtrade/config.json" + # Default command used when running `docker compose up` + command: > + trade + --logfile /freqtrade/user_data/freqtrade.log + --db-url sqlite:////freqtrade/user_data/tradesv3.sqlite + --config /freqtrade/user_data/config.json + --strategy SampleStrategy From f508324fc82641f5ccddc46940bff9baeda4f8ac Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Feb 2020 13:38:45 +0100 Subject: [PATCH 168/317] Update docker documentation to be easier to use --- docs/docker.md | 127 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 3 deletions(-) diff --git a/docs/docker.md b/docs/docker.md index d1684abc5..b1eb0b298 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -1,4 +1,4 @@ -# Using FreqTrade with Docker +# Using Freqtrade with Docker ## Install Docker @@ -8,13 +8,134 @@ Start by downloading and installing Docker CE for your platform: * [Windows](https://docs.docker.com/docker-for-windows/install/) * [Linux](https://docs.docker.com/install/) +Optionally, [docker-compose](https://docs.docker.com/compose/install/) should be installed and available to follow the docker quick start guide. + Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. -## Download the official FreqTrade docker image +## Freqtrade with docker-compose + +Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage. + +!!! Note + The following section assumes that docker and docker-compose is installed and available to the logged in user. + +!!! Note + All below comands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file. + +### Docker quick start + +Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory. + +``` bash +mkdir freqtrade +cd freqtrade/ +# Download the docker-compose file from the repository +curl https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docker-compose.yml -o docker-compose.yml + +# Pull the freqtrade image +docker-compose pull + +# Create user directory structure +docker-compose run --rm freqtrade create-userdir --userdir user_data + +# Create configuration - Requires answering interactive questions +docker-compose run --rm freqtrade new-config --config user_data/config.json +``` + +The above snippet will create a directory called "freqtrade" - download the latest compose file and pull the freqtrade image. +The last 2 steps will create the user-directory, as well as a default configuration based on your selections. + +#### Adding your strategy + +The configuration is now available as `user_data/config.json`. +You should now copy your strategy to `user_data/strategies/` - and add the Strategy class name to the `docker-compose.yml` file, replacing `SampleStrategy`. + +Once this is done, you're ready to launch the bot in trading mode. + +``` bash +docker-compose up -d +``` + +#### Docker-compose logs + +Logs will be written to `user_data/freqtrade.log`. +Alternatively, you can check the latest logs using `docker-compose logs -f`. + +#### Database + +The database will be in the user_data directory as well, and will be called `user_data/tradesv3.sqlite`. + +#### Updating freqtrade with docker-compose + +To update freqtrade when using docker-compose is as simple as running the following 2 commands: + +``` bash +# Download the latest image +docker-compose pull +# Restart the image +docker-compose up -d +``` + +This will first pull the latest image, and will then restart the container with the just pulled version. + +!!! Note + You should always check the changelog for breaking changes / manual interventions required and make sure the bot starts correctly after the update. + +#### Going from here + +Advanced users may edit the docker-compose file further to include all possible options or arguments. + +All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. + +!!! Note "`docker-compose run --rm`" + Inluding `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (`freqtrade trade`). + +##### Example: Download data with docker-compose + +Downloading backtest data for one pair from binance. The data will be stored in the host directory `user_data/data/`. + +``` bash +docker-compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h +``` + +Head over to the [Data Downloading Documentation](data-download.md) for more details on downloading data. + +##### Example: Backtest with docker-compose + +Backtesting in docker-containers: + +``` bash +docker-compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m +``` + +Head over to the [Backtesting Documentation](backtesting.md) to learn more. + +#### Additional dependencies with docker-compose + +If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host. +For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) for an example). + +You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions. + +``` yaml + image: freqtrade_custom + build: + context: . + dockerfile: "./Dockerfile." +``` + +You can then run `docker-compose build` to build the docker image, and run it using the commands described above. + +## Docker - without docker compose + +!!! Warning + The below documentation is provided for completeness and assumes that you are somewhat familiar with running docker containers. If you're just starting out with docker, we recommend to follow the [Freqtrade with docker-compose](#freqtrade-with-docker-compose) instructions. + +### Download the official Freqtrade docker image Pull the image from docker hub. -Branches / tags available can be checked out on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). +Branches / tags available can be checked out on [Dockerhub tags page](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). ```bash docker pull freqtradeorg/freqtrade:develop From 52f4187129566deeed3121752d34e5131410a87e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Feb 2020 13:51:55 +0100 Subject: [PATCH 169/317] Allow exchange templates to configure outside-options too --- freqtrade/commands/build_config_commands.py | 2 +- freqtrade/templates/base_config.json.j2 | 4 +- .../subtemplates/exchange_binance.j2 | 80 ++++++++++--------- .../subtemplates/exchange_generic.j2 | 24 +++--- .../templates/subtemplates/exchange_kraken.j2 | 67 ++++++++-------- 5 files changed, 91 insertions(+), 86 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 838fd510a..7dd1be607 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -90,10 +90,10 @@ def ask_user_config() -> Dict[str, Any]: "name": "exchange_name", "message": "Select exchange", "choices": [ - "bittrex", "binance", "binanceje", "binanceus", + "bittrex", "kraken", Separator(), "other", diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index 1370bfa80..0a4f92c4b 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -27,9 +27,7 @@ "sell_profit_only": false, "ignore_roi_if_buy_signal": false }, - "exchange": { - {{ exchange | indent(8) }} - }, + {{ exchange | indent(4) }}, "pairlists": [ {"method": "StaticPairList"} ], diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index c527d296b..03aa0560c 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -1,39 +1,41 @@ -"name": "{{ exchange_name | lower }}", -"key": "{{ exchange_key }}", -"secret": "{{ exchange_secret }}", -"ccxt_config": {"enableRateLimit": true}, -"ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 200 -}, -"pair_whitelist": [ - "ALGO/BTC", - "ATOM/BTC", - "BAT/BTC", - "BCH/BTC", - "BRD/BTC", - "EOS/BTC", - "ETH/BTC", - "IOTA/BTC", - "LINK/BTC", - "LTC/BTC", - "NEO/BTC", - "NXS/BTC", - "XMR/BTC", - "XRP/BTC", - "XTZ/BTC" -], -"pair_blacklist": [ - "BNB/BTC", - "BNB/BUSD", - "BNB/ETH", - "BNB/EUR", - "BNB/NGN", - "BNB/PAX", - "BNB/RUB", - "BNB/TRY", - "BNB/TUSD", - "BNB/USDC", - "BNB/USDS", - "BNB/USDT", -] +"exchange": { + "name": "{{ exchange_name | lower }}", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 200 + }, + "pair_whitelist": [ + "ALGO/BTC", + "ATOM/BTC", + "BAT/BTC", + "BCH/BTC", + "BRD/BTC", + "EOS/BTC", + "ETH/BTC", + "IOTA/BTC", + "LINK/BTC", + "LTC/BTC", + "NEO/BTC", + "NXS/BTC", + "XMR/BTC", + "XRP/BTC", + "XTZ/BTC" + ], + "pair_blacklist": [ + "BNB/BTC", + "BNB/BUSD", + "BNB/ETH", + "BNB/EUR", + "BNB/NGN", + "BNB/PAX", + "BNB/RUB", + "BNB/TRY", + "BNB/TUSD", + "BNB/USDC", + "BNB/USDS", + "BNB/USDT", + ] +} diff --git a/freqtrade/templates/subtemplates/exchange_generic.j2 b/freqtrade/templates/subtemplates/exchange_generic.j2 index 33309de3b..ade9c2f28 100644 --- a/freqtrade/templates/subtemplates/exchange_generic.j2 +++ b/freqtrade/templates/subtemplates/exchange_generic.j2 @@ -1,13 +1,15 @@ -"name": "{{ exchange_name | lower }}", -"key": "{{ exchange_key }}", -"secret": "{{ exchange_secret }}", -"ccxt_config": {"enableRateLimit": true}, -"ccxt_async_config": { - "enableRateLimit": true -}, -"pair_whitelist": [ +"exchange": { + "name": "{{ exchange_name | lower }}", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true + }, + "pair_whitelist": [ -], -"pair_blacklist": [ + ], + "pair_blacklist": [ -] + ] +} diff --git a/freqtrade/templates/subtemplates/exchange_kraken.j2 b/freqtrade/templates/subtemplates/exchange_kraken.j2 index 690828887..7139a0830 100644 --- a/freqtrade/templates/subtemplates/exchange_kraken.j2 +++ b/freqtrade/templates/subtemplates/exchange_kraken.j2 @@ -1,33 +1,36 @@ -"name": "kraken", -"key": "{{ exchange_key }}", -"secret": "{{ exchange_secret }}", -"ccxt_config": {"enableRateLimit": true}, -"ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 1000 -}, -"pair_whitelist": [ - "ADA/EUR", - "ATOM/EUR", - "BAT/EUR", - "BCH/EUR", - "BTC/EUR", - "DAI/EUR", - "DASH/EUR", - "EOS/EUR", - "ETC/EUR", - "ETH/EUR", - "LINK/EUR", - "LTC/EUR", - "QTUM/EUR", - "REP/EUR", - "WAVES/EUR", - "XLM/EUR", - "XMR/EUR", - "XRP/EUR", - "XTZ/EUR", - "ZEC/EUR" -], -"pair_blacklist": [ +"download_trades": true, +"exchange": { + "name": "kraken", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 1000 + }, + "pair_whitelist": [ + "ADA/EUR", + "ATOM/EUR", + "BAT/EUR", + "BCH/EUR", + "BTC/EUR", + "DAI/EUR", + "DASH/EUR", + "EOS/EUR", + "ETC/EUR", + "ETH/EUR", + "LINK/EUR", + "LTC/EUR", + "QTUM/EUR", + "REP/EUR", + "WAVES/EUR", + "XLM/EUR", + "XMR/EUR", + "XRP/EUR", + "XTZ/EUR", + "ZEC/EUR" + ], + "pair_blacklist": [ -] + ] +} From 34f04668c19f719f712b34bd556f13c5fb3af1bc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Feb 2020 14:02:51 +0100 Subject: [PATCH 170/317] Add template for bittrex --- docs/utils.md | 1 - .../subtemplates/exchange_bittrex.j2 | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 freqtrade/templates/subtemplates/exchange_bittrex.j2 diff --git a/docs/utils.md b/docs/utils.md index 66101d9bc..a986f040b 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -40,7 +40,6 @@ optional arguments: Creates a new configuration file, asking some questions which are important selections for a configuration. - ``` usage: freqtrade new-config [-h] [-c PATH] diff --git a/freqtrade/templates/subtemplates/exchange_bittrex.j2 b/freqtrade/templates/subtemplates/exchange_bittrex.j2 new file mode 100644 index 000000000..7a7e8e291 --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_bittrex.j2 @@ -0,0 +1,24 @@ +"exchange": { + "name": "{{ exchange_name | lower }}", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 500 + }, + "pair_whitelist": [ + "ETH/BTC", + "LTC/BTC", + "ETC/BTC", + "DASH/BTC", + "ZEC/BTC", + "XLM/BTC", + "XRP/BTC", + "TRX/BTC", + "ADA/BTC", + "XMR/BTC" + ], + "pair_blacklist": [ + ] +} From c4031761fec25cb1193557cc09733451fa028612 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Feb 2020 09:53:20 +0100 Subject: [PATCH 171/317] Don't validate exchange for data-download subcommand --- freqtrade/commands/data_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index ddc2ca25b..aee144505 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -37,7 +37,7 @@ def start_download_data(args: Dict[str, Any]) -> None: pairs_not_available: List[str] = [] # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) try: if config.get('download_trades'): From f3b1161640507d91e7694f03e80095304bfdaf0a Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 8 Feb 2020 21:02:52 +0100 Subject: [PATCH 172/317] wide notifications fixes --- docs/configuration.md | 2 + docs/telegram-usage.md | 2 +- docs/webhook-config.md | 51 ++++++++++++ freqtrade/constants.py | 4 +- freqtrade/freqtradebot.py | 137 ++++++++++++++++++++++++--------- freqtrade/rpc/rpc.py | 14 ++-- freqtrade/rpc/telegram.py | 20 +++-- freqtrade/rpc/webhook.py | 4 + tests/rpc/test_rpc.py | 6 +- tests/rpc/test_rpc_telegram.py | 81 ++++++++++++++++--- tests/rpc/test_rpc_webhook.py | 46 ++++++----- tests/test_freqtradebot.py | 6 +- 12 files changed, 288 insertions(+), 85 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c0404d647..53e554709 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -92,7 +92,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `webhook.enabled` | Enable usage of Webhook notifications
***Datatype:*** *Boolean* | `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* | `webhook.webhookbuy` | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* +| `webhook.webhookbuycancel` | Payload to send on buy order cancel. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* | `webhook.webhooksell` | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* +| `webhook.webhooksellcancel` | Payload to send on sell order cancel. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* | `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* | `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Boolean* | `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *IPv4* diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index ed0c21a6e..ac9cea3d6 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -55,7 +55,7 @@ official commands. You can ask at any moment for help with `/help`. | `/reload_conf` | | Reloads the configuration file | `/show_config` | | Shows part of the current configuration with relevant settings to operation | `/status` | | Lists all open trades -| `/status table` | | List all open trades in a table format +| `/status table` | | List all open trades in a table format. Pending buy orders are marked with an asterisk(*) | `/count` | | Displays number of trades used and available | `/profit` | | Display a summary of your profit/loss from close trades and some stats about your performance | `/forcesell ` | | Instantly sells the given trade (Ignoring `minimum_roi`). diff --git a/docs/webhook-config.md b/docs/webhook-config.md index 9e0a34eae..878b18e8a 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -15,11 +15,21 @@ Sample configuration (tested using IFTTT). "value2": "limit {limit:8f}", "value3": "{stake_amount:8f} {stake_currency}" }, + "webhookbuycancel": { + "value1": "Cancelling Buy {pair}", + "value2": "limit {limit:8f}", + "value3": "{stake_amount:8f} {stake_currency}" + }, "webhooksell": { "value1": "Selling {pair}", "value2": "limit {limit:8f}", "value3": "profit: {profit_amount:8f} {stake_currency}" }, + "webhooksellcancel": { + "value1": "Cancelling Sell {pair}", + "value2": "limit {limit:8f}", + "value3": "profit: {profit_amount:8f} {stake_currency}" + }, "webhookstatus": { "value1": "Status: {status}", "value2": "", @@ -40,10 +50,29 @@ Possible parameters are: * `exchange` * `pair` * `limit` +* `amount` * `stake_amount` * `stake_currency` * `fiat_currency` * `order_type` +* `open_rate` +* `current_rate` + +### Webhookbuycancel + +The fields in `webhook.webhookbuycancel` are filled when the bot cancels a buy order. Parameters are filled using string.format. +Possible parameters are: + +* `exchange` +* `pair` +* `limit` +* `amount` +* `stake_amount` +* `stake_currency` +* `fiat_currency` +* `order_type` +* `open_rate` +* `current_rate` ### Webhooksell @@ -57,6 +86,7 @@ Possible parameters are: * `amount` * `open_rate` * `current_rate` +* `close_rate` * `profit_amount` * `profit_percent` * `stake_currency` @@ -66,6 +96,27 @@ Possible parameters are: * `open_date` * `close_date` +### Webhooksellcancel + +The fields in `webhook.webhooksellcancel` are filled when the bot cancels a sell order. Parameters are filled using string.format. +Possible parameters are: + +* `exchange` +* `pair` +* `gain` +* `limit` +* `amount` +* `open_rate` +* `current_rate` +* `close_rate` +* `profit_amount` +* `profit_percent` +* `stake_currency` +* `fiat_currency` +* `sell_reason` +* `order_type` +* `open_date` + ### Webhookstatus The fields in `webhook.webhookstatus` are used for regular status messages (Started / Stopped / ...). Parameters are filled using string.format. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e68e741af..b34805e94 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -78,7 +78,7 @@ CONF_SCHEMA = { 'amend_last_stake_amount': {'type': 'boolean', 'default': False}, 'last_stake_amount_min_ratio': { 'type': 'number', 'minimum': 0.0, 'maximum': 1.0, 'default': 0.5 - }, + }, 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, 'dry_run_wallet': {'type': 'number', 'default': DRY_RUN_WALLET}, @@ -191,7 +191,9 @@ CONF_SCHEMA = { 'properties': { 'enabled': {'type': 'boolean'}, 'webhookbuy': {'type': 'object'}, + 'webhookbuycancel': {'type': 'object'}, 'webhooksell': {'type': 'object'}, + 'webhooksellcancel': {'type': 'object'}, 'webhookstatus': {'type': 'object'}, }, }, diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e51b3d550..2f57ca41b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -234,7 +234,7 @@ class FreqtradeBot: return trades_created - def get_buy_rate(self, pair: str, tick: Dict = None) -> float: + def get_buy_rate(self, pair: str, refresh: bool = False, tick: Dict = None) -> float: """ Calculates bid target between current ask price and last price :return: float: Price @@ -253,7 +253,7 @@ class FreqtradeBot: else: if not tick: logger.info('Using Last Ask / Last Price') - ticker = self.exchange.fetch_ticker(pair) + ticker = self.exchange.fetch_ticker(pair, refresh) else: ticker = tick if ticker['ask'] < ticker['last']: @@ -404,7 +404,7 @@ class FreqtradeBot: stake_amount = self.get_trade_stake_amount(pair) if not stake_amount: - logger.debug("Stake amount is 0, ignoring possible trade for {pair}.") + logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.") return False logger.info(f"Buy signal found: about create a new trade with stake_amount: " @@ -414,10 +414,12 @@ class FreqtradeBot: if ((bid_check_dom.get('enabled', False)) and (bid_check_dom.get('bids_to_ask_delta', 0) > 0)): if self._check_depth_of_market_buy(pair, bid_check_dom): + logger.info(f'Executed Buy for {pair}.') return self.execute_buy(pair, stake_amount) else: return False + logger.info(f'Executed Buy for {pair}') return self.execute_buy(pair, stake_amount) else: return False @@ -450,7 +452,7 @@ class FreqtradeBot: """ Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY - :return: None + :return: bool """ time_in_force = self.strategy.order_time_in_force['buy'] @@ -458,7 +460,7 @@ class FreqtradeBot: buy_limit_requested = price else: # Calculate price - buy_limit_requested = self.get_buy_rate(pair) + buy_limit_requested = self.get_buy_rate(pair, True) min_stake_amount = self._get_min_pair_stake_amount(pair, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: @@ -547,11 +549,37 @@ class FreqtradeBot: 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, - 'limit': trade.open_rate, + 'limit': trade.open_rate_requested, 'order_type': order_type, 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency', None), + 'amount': trade.amount, + 'open_date': trade.open_date or datetime.utcnow(), + 'current_rate': trade.open_rate_requested, + } + + # Send the message + self.rpc.send_msg(msg) + + def _notify_buy_cancel(self, trade: Trade, order_type: str) -> None: + """ + Sends rpc notification when a buy cancel occured. + """ + current_rate = self.get_buy_rate(trade.pair, False) + + msg = { + 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, + 'exchange': self.exchange.name.capitalize(), + 'pair': trade.pair, + 'limit': trade.open_rate_requested, + 'order_type': order_type, + 'stake_amount': trade.stake_amount, + 'stake_currency': self.config['stake_currency'], + 'fiat_currency': self.config.get('fiat_display_currency', None), + 'amount': trade.amount, + 'open_date': trade.open_date, + 'current_rate': current_rate, } # Send the message @@ -587,7 +615,7 @@ class FreqtradeBot: return trades_closed - def get_sell_rate(self, pair: str, refresh: bool) -> float: + def get_sell_rate(self, pair: str, refresh: bool = False) -> float: """ Get sell rate - either using get-ticker bid or first bid based on orderbook The orderbook portion is only used for rpc messaging, which would otherwise fail @@ -751,7 +779,7 @@ class FreqtradeBot: update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60) if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat: # cancelling the current stoploss on exchange first - logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s})' + logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s}) ' 'in order to add another one ...', order['id']) try: self.exchange.cancel_order(order['id'], trade.pair) @@ -777,7 +805,7 @@ class FreqtradeBot: if should_sell.sell_flag: self.execute_sell(trade, sell_rate, should_sell.sell_type) - logger.info('executed sell, reason: %s', should_sell.sell_type) + logger.info(f'Executed Sell for {trade.pair}. Reason: {should_sell.sell_type}') return True return False @@ -820,41 +848,41 @@ class FreqtradeBot: if ((order['side'] == 'buy' and order['status'] == 'canceled') or (self._check_timed_out('buy', order))): - self.handle_timedout_limit_buy(trade, order) self.wallets.update() + order_type = self.strategy.order_types['buy'] + self._notify_buy_cancel(trade, order_type) elif ((order['side'] == 'sell' and order['status'] == 'canceled') or (self._check_timed_out('sell', order))): self.handle_timedout_limit_sell(trade, order) self.wallets.update() + order_type = self.strategy.order_types['sell'] + self._notify_sell_cancel(trade, order_type) - def handle_buy_order_full_cancel(self, trade: Trade, reason: str) -> None: - """Close trade in database and send message""" + def delete_trade(self, trade: Trade) -> None: + """Delete trade in database""" Trade.session.delete(trade) Trade.session.flush() - logger.info('Buy order %s for %s.', reason, trade) - self.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Unfilled buy order for {trade.pair} {reason}' - }) def handle_timedout_limit_buy(self, trade: Trade, order: Dict) -> bool: """ Buy timeout - cancel order :return: True if order was fully cancelled """ - reason = "cancelled due to timeout" if order['status'] != 'canceled': + reason = "cancelled due to timeout" corder = self.exchange.cancel_order(trade.open_order_id, trade.pair) + logger.info('Buy order %s for %s.', reason, trade) else: # Order was cancelled already, so we can reuse the existing dict corder = order - reason = "canceled on Exchange" + reason = "cancelled on exchange" + logger.info('Buy order %s for %s.', reason, trade) if corder.get('remaining', order['remaining']) == order['amount']: # if trade is not partially completed, just delete the trade - self.handle_buy_order_full_cancel(trade, reason) + self.delete_trade(trade) return True # if trade is partially complete, edit the stake details for the trade @@ -878,10 +906,6 @@ class FreqtradeBot: trade.open_order_id = None logger.info('Partial buy order timeout for %s.', trade) - self.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Remaining buy order for {trade.pair} cancelled due to timeout' - }) return False def handle_timedout_limit_sell(self, trade: Trade, order: Dict) -> bool: @@ -889,24 +913,22 @@ class FreqtradeBot: Sell timeout - cancel order and update trade :return: True if order was fully cancelled """ + # if trade is not partially completed, just cancel the trade if order['remaining'] == order['amount']: - # if trade is not partially completed, just cancel the trade if order["status"] != "canceled": - reason = "due to timeout" + reason = "cancelled due to timeout" + # if trade is not partially completed, just delete the trade self.exchange.cancel_order(trade.open_order_id, trade.pair) - logger.info('Sell order timeout for %s.', trade) + logger.info('Sell order %s for %s.', reason, trade) else: - reason = "on exchange" - logger.info('Sell order canceled on exchange for %s.', trade) + reason = "cancelled on exchange" + logger.info('Sell order %s for %s.', reason, trade) + trade.close_rate = None trade.close_profit = None trade.close_date = None trade.is_open = True trade.open_order_id = None - self.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Unfilled sell order for {trade.pair} cancelled {reason}' - }) return True @@ -938,13 +960,13 @@ class FreqtradeBot: raise DependencyException( f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}") - def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> None: + def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> bool: """ Executes a limit sell for the given trade and limit :param trade: Trade instance :param limit: limit rate for the sell order :param sellreason: Reason the sell was triggered - :return: None + :return: bool """ sell_type = 'sell' if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): @@ -965,7 +987,7 @@ class FreqtradeBot: order_type = self.strategy.order_types[sell_type] if sell_reason == SellType.EMERGENCY_SELL: - # Emergencysells (default to market!) + # Emergency sells (default to market!) order_type = self.strategy.order_types.get("emergencysell", "market") amount = self._safe_sell_amount(trade.pair, trade.amount) @@ -990,6 +1012,8 @@ class FreqtradeBot: self._notify_sell(trade, order_type) + return True + def _notify_sell(self, trade: Trade, order_type: str) -> None: """ Sends rpc notification when a sell occured. @@ -1006,7 +1030,7 @@ class FreqtradeBot: 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, 'gain': gain, - 'limit': trade.close_rate_requested, + 'limit': profit_rate, 'order_type': order_type, 'amount': trade.amount, 'open_rate': trade.open_rate, @@ -1017,6 +1041,45 @@ class FreqtradeBot: 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.utcnow(), 'stake_currency': self.config['stake_currency'], + 'fiat_currency': self.config.get('fiat_display_currency', None), + } + + if 'fiat_display_currency' in self.config: + msg.update({ + 'fiat_currency': self.config['fiat_display_currency'], + }) + + # Send the message + self.rpc.send_msg(msg) + + def _notify_sell_cancel(self, trade: Trade, order_type: str) -> None: + """ + Sends rpc notification when a sell cancel occured. + """ + profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested + profit_trade = trade.calc_profit(rate=profit_rate) + # Use cached ticker here - it was updated seconds ago. + current_rate = self.get_sell_rate(trade.pair, False) + profit_percent = trade.calc_profit_ratio(profit_rate) + gain = "profit" if profit_percent > 0 else "loss" + + msg = { + 'type': RPCMessageType.SELL_CANCEL_NOTIFICATION, + 'exchange': trade.exchange.capitalize(), + 'pair': trade.pair, + 'gain': gain, + 'limit': profit_rate, + 'order_type': order_type, + 'amount': trade.amount, + 'open_rate': trade.open_rate, + 'current_rate': current_rate, + 'profit_amount': profit_trade, + 'profit_percent': profit_percent, + 'sell_reason': trade.sell_reason, + 'open_date': trade.open_date, + 'close_date': trade.close_date, + 'stake_currency': self.config['stake_currency'], + 'fiat_currency': self.config.get('fiat_display_currency', None), } if 'fiat_display_currency' in self.config: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 7f5cfc101..c1efea79e 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -26,7 +26,9 @@ class RPCMessageType(Enum): WARNING_NOTIFICATION = 'warning' CUSTOM_NOTIFICATION = 'custom' BUY_NOTIFICATION = 'buy' + BUY_CANCEL_NOTIFICATION = 'buy_cancel' SELL_NOTIFICATION = 'sell' + SELL_CANCEL_NOTIFICATION = 'sell_cancel' def __repr__(self): return self.value @@ -39,6 +41,7 @@ class RPCException(Exception): raise RPCException('*Status:* `no active trade`') """ + def __init__(self, message: str) -> None: super().__init__(self) self.message = message @@ -157,15 +160,16 @@ class RPC: profit_str = f'{trade_perc:.2f}%' if self._fiat_converter: fiat_profit = self._fiat_converter.convert_amount( - trade_profit, - stake_currency, - fiat_display_currency - ) + trade_profit, + stake_currency, + fiat_display_currency + ) if fiat_profit and not isnan(fiat_profit): profit_str += f" ({fiat_profit:.2f})" trades_list.append([ trade.id, - trade.pair, + trade.pair + ['', '*'][trade.open_order_id is not None + and trade.close_rate_requested is None], shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), profit_str ]) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e9ecdcff6..0dd7a8ffd 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -134,13 +134,18 @@ class Telegram(RPC): msg['stake_amount_fiat'] = 0 message = ("*{exchange}:* Buying {pair}\n" - "at rate `{limit:.8f}\n" - "({stake_amount:.6f} {stake_currency}").format(**msg) + "*Amount:* `{amount:.8f}`\n" + "*Open Rate:* `{limit:.8f}`\n" + "*Current Rate:* `{current_rate:.8f}`\n" + "*Total:* `({stake_amount:.6f} {stake_currency}").format(**msg) if msg.get('fiat_currency', None): - message += ",{stake_amount_fiat:.3f} {fiat_currency}".format(**msg) + message += ", {stake_amount_fiat:.3f} {fiat_currency}".format(**msg) message += ")`" + elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION: + message = "*{exchange}:* Cancelling Buy {pair}".format(**msg) + elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: msg['amount'] = round(msg['amount'], 8) msg['profit_percent'] = round(msg['profit_percent'] * 100, 2) @@ -149,10 +154,10 @@ class Telegram(RPC): msg['duration_min'] = msg['duration'].total_seconds() / 60 message = ("*{exchange}:* Selling {pair}\n" - "*Rate:* `{limit:.8f}`\n" "*Amount:* `{amount:.8f}`\n" "*Open Rate:* `{open_rate:.8f}`\n" "*Current Rate:* `{current_rate:.8f}`\n" + "*Close Rate:* `{limit:.8f}`\n" "*Sell Reason:* `{sell_reason}`\n" "*Duration:* `{duration} ({duration_min:.1f} min)`\n" "*Profit:* `{profit_percent:.2f}%`").format(**msg) @@ -163,8 +168,11 @@ class Telegram(RPC): and self._fiat_converter): msg['profit_fiat'] = self._fiat_converter.convert_amount( msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) - message += ('` ({gain}: {profit_amount:.8f} {stake_currency}`' - '` / {profit_fiat:.3f} {fiat_currency})`').format(**msg) + message += (' `({gain}: {profit_amount:.8f} {stake_currency}' + ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg) + + elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION: + message = "*{exchange}:* Cancelling Sell {pair}".format(**msg) elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: message = '*Status:* `{status}`'.format(**msg) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 37ca466de..1309663d4 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -41,8 +41,12 @@ class Webhook(RPC): if msg['type'] == RPCMessageType.BUY_NOTIFICATION: valuedict = self._config['webhook'].get('webhookbuy', None) + elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION: + valuedict = self._config['webhook'].get('webhookbuycancel', None) elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: valuedict = self._config['webhook'].get('webhooksell', None) + elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION: + valuedict = self._config['webhook'].get('webhooksellcancel', None) elif msg['type'] in(RPCMessageType.STATUS_NOTIFICATION, RPCMessageType.CUSTOM_NOTIFICATION, RPCMessageType.WARNING_NOTIFICATION): diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 36fce1797..a35bfa0d6 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -122,7 +122,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert "Since" in headers assert "Pair" in headers assert 'instantly' == result[0][2] - assert 'ETH/BTC' == result[0][1] + assert 'ETH/BTC' in result[0][1] assert '-0.59%' == result[0][3] # Test with fiatconvert @@ -131,7 +131,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert "Since" in headers assert "Pair" in headers assert 'instantly' == result[0][2] - assert 'ETH/BTC' == result[0][1] + assert 'ETH/BTC' in result[0][1] assert '-0.59% (-0.09)' == result[0][3] mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', @@ -140,7 +140,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: rpc._freqtrade.exchange._cached_ticker = {} result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] - assert 'ETH/BTC' == result[0][1] + assert 'ETH/BTC' in result[0][1] assert 'nan%' == result[0][3] diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index ffc29ee12..ae9c0c4dc 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -284,7 +284,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: fields = re.sub('[ ]+', ' ', line[2].strip()).split(' ') assert int(fields[0]) == 1 - assert fields[1] == 'ETH/BTC' + assert 'ETH/BTC' in fields[1] assert msg_mock.call_count == 1 @@ -1200,12 +1200,35 @@ def test_send_msg_buy_notification(default_conf, mocker) -> None: 'stake_amount': 0.001, 'stake_amount_fiat': 0.0, 'stake_currency': 'BTC', - 'fiat_currency': 'USD' + 'fiat_currency': 'USD', + 'current_rate': 1.099e-05, + 'amount': 1333.3333333333335, + 'open_date': arrow.utcnow().shift(hours=-1) }) assert msg_mock.call_args[0][0] \ == '*Bittrex:* Buying ETH/BTC\n' \ - 'at rate `0.00001099\n' \ - '(0.001000 BTC,0.000 USD)`' + '*Amount:* `1333.33333333`\n' \ + '*Open Rate:* `0.00001099`\n' \ + '*Current Rate:* `0.00001099`\n' \ + '*Total:* `(0.001000 BTC, 0.000 USD)`' + + +def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + }) + assert msg_mock.call_args[0][0] \ + == ('*Bittrex:* Cancelling Buy ETH/BTC') def test_send_msg_sell_notification(default_conf, mocker) -> None: @@ -1239,13 +1262,13 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] \ == ('*Binance:* Selling KEY/ETH\n' - '*Rate:* `0.00003201`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' + '*Close Rate:* `0.00003201`\n' '*Sell Reason:* `stop_loss`\n' '*Duration:* `1:00:00 (60.0 min)`\n' - '*Profit:* `-57.41%`` (loss: -0.05746268 ETH`` / -24.812 USD)`') + '*Profit:* `-57.41%` `(loss: -0.05746268 ETH / -24.812 USD)`') msg_mock.reset_mock() telegram.send_msg({ @@ -1267,10 +1290,10 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] \ == ('*Binance:* Selling KEY/ETH\n' - '*Rate:* `0.00003201`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' + '*Close Rate:* `0.00003201`\n' '*Sell Reason:* `stop_loss`\n' '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n' '*Profit:* `-57.41%`') @@ -1278,6 +1301,37 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: telegram._fiat_converter.convert_amount = old_convamount +def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + old_convamount = telegram._fiat_converter.convert_amount + telegram._fiat_converter.convert_amount = lambda a, b, c: -24.812 + telegram.send_msg({ + 'type': RPCMessageType.SELL_CANCEL_NOTIFICATION, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + }) + assert msg_mock.call_args[0][0] \ + == ('*Binance:* Cancelling Sell KEY/ETH') + + msg_mock.reset_mock() + telegram.send_msg({ + 'type': RPCMessageType.SELL_CANCEL_NOTIFICATION, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + }) + assert msg_mock.call_args[0][0] \ + == ('*Binance:* Cancelling Sell KEY/ETH') + # Reset singleton function to avoid random breaks + telegram._fiat_converter.convert_amount = old_convamount + + def test_send_msg_status_notification(default_conf, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( @@ -1360,12 +1414,17 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None: 'stake_amount': 0.001, 'stake_amount_fiat': 0.0, 'stake_currency': 'BTC', - 'fiat_currency': None + 'fiat_currency': None, + 'current_rate': 1.099e-05, + 'amount': 1333.3333333333335, + 'open_date': arrow.utcnow().shift(hours=-1) }) assert msg_mock.call_args[0][0] \ == '*Bittrex:* Buying ETH/BTC\n' \ - 'at rate `0.00001099\n' \ - '(0.001000 BTC)`' + '*Amount:* `1333.33333333`\n' \ + '*Open Rate:* `0.00001099`\n' \ + '*Current Rate:* `0.00001099`\n' \ + '*Total:* `(0.001000 BTC)`' def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: @@ -1398,10 +1457,10 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] \ == '*Binance:* Selling KEY/ETH\n' \ - '*Rate:* `0.00003201`\n' \ '*Amount:* `1333.33333333`\n' \ '*Open Rate:* `0.00007500`\n' \ '*Current Rate:* `0.00003201`\n' \ + '*Close Rate:* `0.00003201`\n' \ '*Sell Reason:* `stop_loss`\n' \ '*Duration:* `2:35:03 (155.1 min)`\n' \ '*Profit:* `-57.41%`' diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index c066aa8e7..3b9ce3f0d 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -13,24 +13,34 @@ from tests.conftest import get_patched_freqtradebot, log_has def get_webhook_dict() -> dict: return { - "enabled": True, - "url": "https://maker.ifttt.com/trigger/freqtrade_test/with/key/c764udvJ5jfSlswVRukZZ2/", - "webhookbuy": { - "value1": "Buying {pair}", - "value2": "limit {limit:8f}", - "value3": "{stake_amount:8f} {stake_currency}" - }, - "webhooksell": { - "value1": "Selling {pair}", - "value2": "limit {limit:8f}", - "value3": "profit: {profit_amount:8f} {stake_currency}" - }, - "webhookstatus": { - "value1": "Status: {status}", - "value2": "", - "value3": "" - } - } + "enabled": True, + "url": "https://maker.ifttt.com/trigger/freqtrade_test/with/key/c764udvJ5jfSlswVRukZZ2/", + "webhookbuy": { + "value1": "Buying {pair}", + "value2": "limit {limit:8f}", + "value3": "{stake_amount:8f} {stake_currency}" + }, + "webhookbuycancel": { + "value1": "Cancelling Buy {pair}", + "value2": "limit {limit:8f}", + "value3": "{stake_amount:8f} {stake_currency}" + }, + "webhooksell": { + "value1": "Selling {pair}", + "value2": "limit {limit:8f}", + "value3": "profit: {profit_amount:8f} {stake_currency}" + }, + "webhooksellcancel": { + "value1": "Cancelling Sell {pair}", + "value2": "limit {limit:8f}", + "value3": "profit: {profit_amount:8f} {stake_currency}" + }, + "webhookstatus": { + "value1": "Status: {status}", + "value2": "", + "value3": "" + } + } def test__init__(mocker, default_conf): diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index f334e4eb0..429d3599d 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -300,7 +300,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf # stoploss shoud be hit assert freqtrade.handle_trade(trade) is True - assert log_has('executed sell, reason: SellType.STOP_LOSS', caplog) + assert log_has('Executed Sell for NEO/BTC. Reason: SellType.STOP_LOSS', caplog) assert trade.sell_reason == SellType.STOP_LOSS.value @@ -1964,7 +1964,7 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() nb_trades = len(trades) assert nb_trades == 0 - assert log_has_re("Buy order canceled on Exchange for Trade.*", caplog) + assert log_has_re("Buy order cancelled on exchange for Trade.*", caplog) def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, open_trade, @@ -2045,7 +2045,7 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 assert open_trade.is_open is True - assert log_has_re("Sell order canceled on exchange for Trade.*", caplog) + assert log_has_re("Sell order cancelled on exchange for Trade.*", caplog) def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, From 4fad7a462cc067c6c91aaa5b970de881025a9f07 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 8 Feb 2020 21:19:07 +0100 Subject: [PATCH 173/317] fixes in webhook-config docs --- docs/webhook-config.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/webhook-config.md b/docs/webhook-config.md index 878b18e8a..b287fa71e 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -51,11 +51,11 @@ Possible parameters are: * `pair` * `limit` * `amount` +* `open_date` * `stake_amount` * `stake_currency` * `fiat_currency` * `order_type` -* `open_rate` * `current_rate` ### Webhookbuycancel @@ -67,11 +67,11 @@ Possible parameters are: * `pair` * `limit` * `amount` +* `open_date` * `stake_amount` * `stake_currency` * `fiat_currency` * `order_type` -* `open_rate` * `current_rate` ### Webhooksell @@ -86,7 +86,6 @@ Possible parameters are: * `amount` * `open_rate` * `current_rate` -* `close_rate` * `profit_amount` * `profit_percent` * `stake_currency` @@ -108,7 +107,6 @@ Possible parameters are: * `amount` * `open_rate` * `current_rate` -* `close_rate` * `profit_amount` * `profit_percent` * `stake_currency` @@ -116,6 +114,7 @@ Possible parameters are: * `sell_reason` * `order_type` * `open_date` +* `close_date` ### Webhookstatus From 879b5138228cc2052ab6bff3fd913080686828e6 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 8 Feb 2020 21:31:36 +0100 Subject: [PATCH 174/317] enhanced method description --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2f57ca41b..5f1024f8c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -452,7 +452,7 @@ class FreqtradeBot: """ Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY - :return: bool + :return: True if a buy order is created, false if it fails. """ time_in_force = self.strategy.order_time_in_force['buy'] @@ -966,7 +966,7 @@ class FreqtradeBot: :param trade: Trade instance :param limit: limit rate for the sell order :param sellreason: Reason the sell was triggered - :return: bool + :return: True if it succeeds (supported) False (not supported) """ sell_type = 'sell' if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): From 1a9787ac76d6df1f337c1889ccf3135ae8cdaa13 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Feb 2020 21:53:34 +0100 Subject: [PATCH 175/317] Add validation for data-download relevant settings --- freqtrade/commands/data_commands.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index aee144505..e8e0f06d2 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -38,6 +38,11 @@ def start_download_data(args: Dict[str, Any]) -> None: # Init exchange exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + # Manual validations of relevant settings + exchange.validate_pairs(config['pairs']) + for timeframe in config['timeframes']: + exchange.validate_timeframes(timeframe) + try: if config.get('download_trades'): From 636bd5acb5e8b66dab60879c50343a26dc8e15c6 Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Sat, 8 Feb 2020 23:21:42 +0100 Subject: [PATCH 176/317] Added filter options to "hyperopt-list" in order to easier find epochs. --profitable Select only profitable epochs. --min-avg-time INT Select epochs on above average time. --max-avg-time INT Select epochs on under average time. --min-avg-profit FLOAT Select epochs on above average profit. --min-total-profit FLOAT Select epochs on above total profit. --- docs/utils.md | 24 +++++++++--- freqtrade/commands/arguments.py | 3 +- freqtrade/commands/cli_options.py | 24 ++++++++++++ freqtrade/commands/hyperopt_commands.py | 47 ++++++++++++++++++------ freqtrade/configuration/configuration.py | 12 ++++++ 5 files changed, 91 insertions(+), 19 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index b0559f9cc..71039f174 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -135,15 +135,27 @@ Common arguments: ``` ``` freqtrade list-hyperopts --help -usage: freqtrade list-hyperopts [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] - [--hyperopt-path PATH] [-1] +usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [--best] + [--profitable] [--min-avg-time INT] + [--max-avg-time INT] [--min-avg-profit FLOAT] + [--min-total-profit FLOAT] [--no-color] + [--print-json] [--no-details] optional arguments: -h, --help show this help message and exit - --hyperopt-path PATH Specify additional lookup path for Hyperopt and - Hyperopt Loss functions. - -1, --one-column Print output in one column. + --best Select only best epochs. + --profitable Select only profitable epochs. + --min-avg-time INT Select epochs on above average time. + --max-avg-time INT Select epochs on under average time. + --min-avg-profit FLOAT + Select epochs on above average profit. + --min-total-profit FLOAT + Select epochs on above total profit. + --no-color Disable colorization of hyperopt results. May be + useful if you are redirecting output to a file. + --print-json Print best result detailization in JSON format. + --no-details Do not print best epoch details. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 1931a51be..2d02058f1 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -59,7 +59,8 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "ticker_interval"] -ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "print_colorized", +ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_list_min_avg_time", + "hyperopt_list_max_avg_time", "hyperopt_list_min_avg_profit", "hyperopt_list_min_total_profit", "print_colorized", "print_json", "hyperopt_list_no_details"] ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 6d8d13129..0c6d64691 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -398,6 +398,30 @@ AVAILABLE_CLI_OPTIONS = { help='Select only best epochs.', action='store_true', ), + "hyperopt_list_min_avg_time": Arg( + '--min-avg-time', + help='Select epochs on above average time.', + type=check_int_nonzero, + metavar='INT', + ), + "hyperopt_list_max_avg_time": Arg( + '--max-avg-time', + help='Select epochs on under average time.', + type=check_int_nonzero, + metavar='INT', + ), + "hyperopt_list_min_avg_profit": Arg( + '--min-avg-profit', + help='Select epochs on above average profit.', + type=float, + metavar='FLOAT', + ), + "hyperopt_list_min_total_profit": Arg( + '--min-total-profit', + help='Select epochs on above total profit.', + type=float, + metavar='FLOAT', + ), "hyperopt_list_no_details": Arg( '--no-details', help='Do not print best epoch details.', diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 5c6f25848..8472fcfe1 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -19,13 +19,20 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - only_best = config.get('hyperopt_list_best', False) - only_profitable = config.get('hyperopt_list_profitable', False) print_colorized = config.get('print_colorized', False) print_json = config.get('print_json', False) no_details = config.get('hyperopt_list_no_details', False) no_header = False + filteroptions = { + 'only_best': config.get('hyperopt_list_best', False), + 'only_profitable': config.get('hyperopt_list_profitable', False), + 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', 0), + 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', 0), + 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', 0.0), + 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', 0.0) + } + trials_file = (config['user_data_dir'] / 'hyperopt_results' / 'hyperopt_results.pickle') @@ -33,7 +40,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: trials = Hyperopt.load_previous_results(trials_file) total_epochs = len(trials) - trials = _hyperopt_filter_trials(trials, only_best, only_profitable) + trials = _hyperopt_filter_trials(trials, filteroptions) # TODO: fetch the interval for epochs to print from the cli option epoch_start, epoch_stop = 0, None @@ -44,7 +51,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: try: # Human-friendly indexes used here (starting from 1) for val in trials[epoch_start:epoch_stop]: - Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized) + Hyperopt.print_results_explanation(val, total_epochs, not filteroptions['only_best'], print_colorized) except KeyboardInterrupt: print('User interrupted..') @@ -63,8 +70,14 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - only_best = config.get('hyperopt_list_best', False) - only_profitable = config.get('hyperopt_list_profitable', False) + filteroptions = { + 'only_best': config.get('hyperopt_list_best', False), + 'only_profitable': config.get('hyperopt_list_profitable', False), + 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', 0), + 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', 0), + 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', 0), + 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', 0) + } no_header = config.get('hyperopt_show_no_header', False) trials_file = (config['user_data_dir'] / @@ -74,7 +87,7 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: trials = Hyperopt.load_previous_results(trials_file) total_epochs = len(trials) - trials = _hyperopt_filter_trials(trials, only_best, only_profitable) + trials = _hyperopt_filter_trials(trials, filteroptions) trials_epochs = len(trials) n = config.get('hyperopt_show_index', -1) @@ -97,18 +110,28 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: header_str="Epoch details") -def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List: +def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: """ Filter our items from the list of hyperopt results """ - if only_best: + if filteroptions['only_best']: trials = [x for x in trials if x['is_best']] - if only_profitable: + if filteroptions['only_profitable']: trials = [x for x in trials if x['results_metrics']['profit'] > 0] + if not filteroptions['only_best']: + if filteroptions['filter_min_avg_time'] > 0: + trials = [x for x in trials if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time']] + if filteroptions['filter_max_avg_time'] > 0: + trials = [x for x in trials if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time']] + if filteroptions['filter_min_avg_profit'] > 0: + trials = [x for x in trials if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit']] + if filteroptions['filter_min_total_profit'] > 0: + trials = [x for x in trials if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit']] + logger.info(f"{len(trials)} " + - ("best " if only_best else "") + - ("profitable " if only_profitable else "") + + ("best " if filteroptions['only_best'] else "") + + ("profitable " if filteroptions['only_profitable'] else "") + "epochs found.") return trials diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index a8b7638c8..f7e87f3a1 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -310,6 +310,18 @@ class Configuration: self._args_to_config(config, argname='hyperopt_list_profitable', logstring='Parameter --profitable detected: {}') + self._args_to_config(config, argname='hyperopt_list_min_avg_time', + logstring='Parameter --min-avg-time detected: {}') + + self._args_to_config(config, argname='hyperopt_list_max_avg_time', + logstring='Parameter --max-avg-time detected: {}') + + self._args_to_config(config, argname='hyperopt_list_min_avg_profit', + logstring='Parameter --min-avg-profit detected: {}') + + self._args_to_config(config, argname='hyperopt_list_min_total_profit', + logstring='Parameter --min-total-profit detected: {}') + self._args_to_config(config, argname='hyperopt_list_no_details', logstring='Parameter --no-details detected: {}') From 2796d3d8a0c1af2c297170f030cc5cfbb4b9c924 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sun, 9 Feb 2020 00:11:58 +0100 Subject: [PATCH 177/317] added missing tests to increase coverage --- tests/rpc/test_rpc_webhook.py | 53 +++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 3b9ce3f0d..ab40047c0 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -54,6 +54,9 @@ def test_send_msg(default_conf, mocker): msg_mock = MagicMock() mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + # Test buy + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) msg = { 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': 'Bittrex', @@ -64,8 +67,6 @@ def test_send_msg(default_conf, mocker): 'stake_currency': 'BTC', 'fiat_currency': 'EUR' } - msg_mock = MagicMock() - mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) webhook.send_msg(msg=msg) assert msg_mock.call_count == 1 assert (msg_mock.call_args[0][0]["value1"] == @@ -74,6 +75,27 @@ def test_send_msg(default_conf, mocker): default_conf["webhook"]["webhookbuy"]["value2"].format(**msg)) assert (msg_mock.call_args[0][0]["value3"] == default_conf["webhook"]["webhookbuy"]["value3"].format(**msg)) + # Test buy cancel + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + msg = { + 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'limit': 0.005, + 'stake_amount': 0.8, + 'stake_amount_fiat': 500, + 'stake_currency': 'BTC', + 'fiat_currency': 'EUR' + } + webhook.send_msg(msg=msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhookbuycancel"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhookbuycancel"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhookbuycancel"]["value3"].format(**msg)) # Test sell msg_mock = MagicMock() mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) @@ -100,7 +122,32 @@ def test_send_msg(default_conf, mocker): default_conf["webhook"]["webhooksell"]["value2"].format(**msg)) assert (msg_mock.call_args[0][0]["value3"] == default_conf["webhook"]["webhooksell"]["value3"].format(**msg)) - + # Test sell cancel + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + msg = { + 'type': RPCMessageType.SELL_CANCEL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': "profit", + 'limit': 0.005, + 'amount': 0.8, + 'order_type': 'limit', + 'open_rate': 0.004, + 'current_rate': 0.005, + 'profit_amount': 0.001, + 'profit_percent': 0.20, + 'stake_currency': 'BTC', + 'sell_reason': SellType.STOP_LOSS.value + } + webhook.send_msg(msg=msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhooksellcancel"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhooksellcancel"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhooksellcancel"]["value3"].format(**msg)) for msgtype in [RPCMessageType.STATUS_NOTIFICATION, RPCMessageType.WARNING_NOTIFICATION, RPCMessageType.CUSTOM_NOTIFICATION]: From c96acd6ca02b6e0dc5cd1e28c78ad0b3be648fe1 Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Sun, 9 Feb 2020 00:16:11 +0100 Subject: [PATCH 178/317] Fixed to pass PEP8 --- freqtrade/commands/arguments.py | 7 ++++--- freqtrade/commands/hyperopt_commands.py | 28 ++++++++++++++++++------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 2d02058f1..6d0c16d88 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -59,9 +59,10 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "ticker_interval"] -ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_list_min_avg_time", - "hyperopt_list_max_avg_time", "hyperopt_list_min_avg_profit", "hyperopt_list_min_total_profit", "print_colorized", - "print_json", "hyperopt_list_no_details"] +ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", + "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time", + "hyperopt_list_min_avg_profit", "hyperopt_list_min_total_profit", + "print_colorized", "print_json", "hyperopt_list_no_details"] ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", "print_json", "hyperopt_show_no_header"] diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 8472fcfe1..f5fcc971f 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -32,7 +32,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', 0.0), 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', 0.0) } - + trials_file = (config['user_data_dir'] / 'hyperopt_results' / 'hyperopt_results.pickle') @@ -51,7 +51,8 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: try: # Human-friendly indexes used here (starting from 1) for val in trials[epoch_start:epoch_stop]: - Hyperopt.print_results_explanation(val, total_epochs, not filteroptions['only_best'], print_colorized) + Hyperopt.print_results_explanation(val, total_epochs, + not filteroptions['only_best'], print_colorized) except KeyboardInterrupt: print('User interrupted..') @@ -121,14 +122,27 @@ def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: if not filteroptions['only_best']: if filteroptions['filter_min_avg_time'] > 0: - trials = [x for x in trials if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time']] + trials = [ + x for x in trials + if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] + ] if filteroptions['filter_max_avg_time'] > 0: - trials = [x for x in trials if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time']] + trials = [ + x for x in trials + if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] + ] if filteroptions['filter_min_avg_profit'] > 0: - trials = [x for x in trials if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit']] + trials = [ + x for x in trials + if x['results_metrics']['avg_profit'] + > filteroptions['filter_min_avg_profit'] + ] if filteroptions['filter_min_total_profit'] > 0: - trials = [x for x in trials if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit']] - + trials = [ + x for x in trials + if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] + ] + logger.info(f"{len(trials)} " + ("best " if filteroptions['only_best'] else "") + ("profitable " if filteroptions['only_profitable'] else "") + From b536d501945c502ecc6003f7fea8ffc781f45f02 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Feb 2020 11:41:29 +0100 Subject: [PATCH 179/317] Address PR Review --- freqtrade/commands/build_config_commands.py | 4 ++-- freqtrade/templates/base_config.json.j2 | 2 +- setup.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 7dd1be607..1598fa2ae 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -187,7 +187,7 @@ def start_new_config(args: Dict[str, Any]) -> None: config_path.unlink() else: raise OperationalException( - f"Configuration `{config_path}` already exists. " - "Please use another configuration name or delete the existing configuration.") + f"Configuration file `{config_path}` already exists. " + "Please delete it or use a different configuration file name.") selections = ask_user_config() deploy_new_config(config_path, selections) diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index 0a4f92c4b..88edeb1e8 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -19,7 +19,7 @@ "bids_to_ask_delta": 1 } }, - "ask_strategy":{ + "ask_strategy": { "use_order_book": false, "order_book_min": 1, "order_book_max": 9, diff --git a/setup.sh b/setup.sh index bce2e56cf..e120190ce 100755 --- a/setup.sh +++ b/setup.sh @@ -223,7 +223,7 @@ function config_generator() { function config() { echo "-------------------------" - echo "Please use freqtrade new-config -c config.json to generate a new configuration file." + echo "Please use 'freqtrade new-config -c config.json' to generate a new configuration file." echo "-------------------------" } From c7ba85c2e6c244f861a9e914782d5d5d14f30837 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sun, 9 Feb 2020 14:19:13 +0300 Subject: [PATCH 180/317] Add tip on running order types for Bittrex --- docs/faq.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index 2416beae4..390b35b9b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -45,12 +45,24 @@ the tutorial [here|Testing-new-strategies-with-Hyperopt](bot-usage.md#hyperopt-c You can use the `/forcesell all` command from Telegram. -### I get the message "RESTRICTED_MARKET" +### I'm getting the "RESTRICTED_MARKET" message in the log Currently known to happen for US Bittrex users. Read [the Bittrex section about restricted markets](exchanges.md#restricted-markets) for more information. +### I'm getting the "Exchange Bittrex does not support market orders." message and cannot run my strategy + +As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Probably your strategy was written for another exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of other exchanges. + +To fix it for Bittrex, redefine order types in the configuration file (do this for all order types that are defined as "market" in your strategy): + +``` +"order_types": { + "stoploss": "limit", +} +``` + ### How do I search the bot logs for something? By default, the bot writes its log into stderr stream. This is implemented this way so that you can easily separate the bot's diagnostics messages from Backtesting, Edge and Hyperopt results, output from other various Freqtrade utility subcommands, as well as from the output of your custom `print()`'s you may have inserted into your strategy. So if you need to search the log messages with the grep utility, you need to redirect stderr to stdout and disregard stdout. From c648ec7c0c76e09d57bee0e52bf820bb0d9adf01 Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Sun, 9 Feb 2020 14:18:56 +0100 Subject: [PATCH 181/317] Added test cases and fixed a minor bug --- freqtrade/commands/hyperopt_commands.py | 6 +++ tests/commands/test_commands.py | 59 ++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index f5fcc971f..38e1fa429 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -120,24 +120,30 @@ def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: if filteroptions['only_profitable']: trials = [x for x in trials if x['results_metrics']['profit'] > 0] + print(trials[0]) + if not filteroptions['only_best']: if filteroptions['filter_min_avg_time'] > 0: + trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ x for x in trials if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] ] if filteroptions['filter_max_avg_time'] > 0: + trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ x for x in trials if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] ] if filteroptions['filter_min_avg_profit'] > 0: + trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ x for x in trials if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit'] ] if filteroptions['filter_min_total_profit'] > 0: + trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ x for x in trials if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index c59799190..fb15c3d7f 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -778,7 +778,64 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): assert all(x not in captured.out for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", " 11/12", " 12/12"]) - + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--min-avg-profit", "0.11" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 10/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--min-total-profit", "0.4" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--min-avg-time", "2000" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", + " 8/12", " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--max-avg-time", "1500" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 6/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12" + " 9/12", " 10/12", " 11/12", " 12/12"]) def test_hyperopt_show(mocker, capsys, hyperopt_results): mocker.patch( From eb3783dc0095740ccf973cbec351a67951cfcda5 Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Sun, 9 Feb 2020 14:30:29 +0100 Subject: [PATCH 182/317] Fixed a blank line issue :-( --- tests/commands/test_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index fb15c3d7f..db8a9289a 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -837,6 +837,7 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12" " 9/12", " 10/12", " 11/12", " 12/12"]) + def test_hyperopt_show(mocker, capsys, hyperopt_results): mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.load_previous_results', From c89a32224c2f77fa14aa2244dfcd23ab9f7ea56d Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sun, 9 Feb 2020 18:40:19 +0300 Subject: [PATCH 183/317] Fix SharpeHyperOptLossDaily --- freqtrade/optimize/hyperopt_loss_sharpe_daily.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_loss_sharpe_daily.py b/freqtrade/optimize/hyperopt_loss_sharpe_daily.py index d8ea3c5fe..5a8ebaa11 100644 --- a/freqtrade/optimize/hyperopt_loss_sharpe_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sharpe_daily.py @@ -39,7 +39,8 @@ class SharpeHyperOptLossDaily(IHyperOptLoss): results['profit_percent'] - slippage_per_trade_ratio # create the index within the min_date and end max_date - t_index = date_range(start=min_date, end=max_date, freq=resample_freq) + t_index = date_range(start=min_date, end=max_date, freq=resample_freq, + normalize=True) sum_daily = ( results.resample(resample_freq, on='close_time').agg( From 40abdd26083f20e6fb05a1facae694b91724ba85 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sun, 9 Feb 2020 18:54:04 +0300 Subject: [PATCH 184/317] Suggest changing strategy --- docs/faq.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 390b35b9b..81fd47561 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -55,14 +55,18 @@ Read [the Bittrex section about restricted markets](exchanges.md#restricted-mark As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Probably your strategy was written for another exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of other exchanges. -To fix it for Bittrex, redefine order types in the configuration file (do this for all order types that are defined as "market" in your strategy): +To fix it for Bittrex, redefine order types in the strategy to use "limit" instead of "market": ``` -"order_types": { - "stoploss": "limit", -} + order_types = { + ... + 'stoploss': 'limit', + ... + } ``` +Same fix should be done in the configuration file, if order types are defined in your custom config rather than in the strategy. + ### How do I search the bot logs for something? By default, the bot writes its log into stderr stream. This is implemented this way so that you can easily separate the bot's diagnostics messages from Backtesting, Edge and Hyperopt results, output from other various Freqtrade utility subcommands, as well as from the output of your custom `print()`'s you may have inserted into your strategy. So if you need to search the log messages with the grep utility, you need to redirect stderr to stdout and disregard stdout. From c83da7cadb4db3c0c7d59d4e787d80e7c876e79c Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sun, 9 Feb 2020 19:11:06 +0300 Subject: [PATCH 185/317] Add section about order types into Bittrex Exchange-specific chapter --- docs/exchanges.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/exchanges.md b/docs/exchanges.md index 3c861ce44..06d33d562 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -32,6 +32,10 @@ To download data for the Kraken exchange, using `--dl-trades` is mandatory, othe ## Bittrex +### Order types + +Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faw.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy). + ### Restricted markets Bittrex split its exchange into US and International versions. From cc3f65d069ddaa08ffa81e76a155f471adc37f9b Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sun, 9 Feb 2020 19:45:04 +0300 Subject: [PATCH 186/317] Fix typo --- docs/exchanges.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 06d33d562..f615bc61a 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -34,7 +34,7 @@ To download data for the Kraken exchange, using `--dl-trades` is mandatory, othe ### Order types -Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faw.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy). +Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faq.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy). ### Restricted markets From 5bf4c5869b81c39176efc7fe6705cebb02a7f489 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sun, 9 Feb 2020 19:32:09 +0100 Subject: [PATCH 187/317] Update hyperopt_commands.py Missed a debug print --- freqtrade/commands/hyperopt_commands.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 38e1fa429..cdfdb5ca6 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -120,8 +120,6 @@ def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: if filteroptions['only_profitable']: trials = [x for x in trials if x['results_metrics']['profit'] > 0] - print(trials[0]) - if not filteroptions['only_best']: if filteroptions['filter_min_avg_time'] > 0: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] From f7c74e551fa4ff0674576de73a7892fdd32d1bfb Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sun, 9 Feb 2020 21:56:59 +0300 Subject: [PATCH 188/317] Fix wording --- docs/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index 81fd47561..94818964b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -53,7 +53,7 @@ Read [the Bittrex section about restricted markets](exchanges.md#restricted-mark ### I'm getting the "Exchange Bittrex does not support market orders." message and cannot run my strategy -As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Probably your strategy was written for another exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of other exchanges. +As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Probably your strategy was written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex). To fix it for Bittrex, redefine order types in the strategy to use "limit" instead of "market": From 4af25ec315c95a9335285d9375b544f0f23b1a46 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Mon, 10 Feb 2020 05:52:07 +0300 Subject: [PATCH 189/317] Adjust mypy and flake commands --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a087103c6..a4a1a29f8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ pytest tests/test_.py::test_ #### Run Flake8 ```bash -flake8 freqtrade +flake8 freqtrade tests ``` We receive a lot of code that fails the `flake8` checks. @@ -61,7 +61,7 @@ Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using #### Run mypy ``` bash -mypy freqtrade +mypy freqtrade tests ``` ## (Core)-Committer Guide From 90ee82ac437cbe1711d70d3663e986a8acce2fe8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2020 08:01:42 +0000 Subject: [PATCH 190/317] Bump ccxt from 1.22.30 to 1.22.39 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.22.30 to 1.22.39. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.22.30...1.22.39) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 466880950..e1ae4a5bd 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.22.30 +ccxt==1.22.39 SQLAlchemy==1.3.13 python-telegram-bot==12.3.0 arrow==0.15.5 From 88f2ad1eae978f9e95e4b20c5c262dc5f97fb298 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2020 08:02:07 +0000 Subject: [PATCH 191/317] Bump pandas from 1.0.0 to 1.0.1 Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.0.0 to 1.0.1. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v1.0.0...v1.0.1) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 21be02a87..68024f587 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ -r requirements-common.txt numpy==1.18.1 -pandas==1.0.0 +pandas==1.0.1 From 6b4094fd92866065ca0717f330e64dc29c5305b1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2020 08:02:45 +0000 Subject: [PATCH 192/317] Bump mkdocs-material from 4.6.0 to 4.6.2 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 4.6.0 to 4.6.2. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/4.6.0...4.6.2) Signed-off-by: dependabot-preview[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 3e53c15e3..3980ecd64 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,2 +1,2 @@ -mkdocs-material==4.6.0 +mkdocs-material==4.6.2 mdx_truly_sane_lists==1.2 From 550f9fc8915841eef544bb51302d4c10553e2794 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2020 08:45:27 +0000 Subject: [PATCH 193/317] Bump python-telegram-bot from 12.3.0 to 12.4.1 Bumps [python-telegram-bot](https://github.com/python-telegram-bot/python-telegram-bot) from 12.3.0 to 12.4.1. - [Release notes](https://github.com/python-telegram-bot/python-telegram-bot/releases) - [Changelog](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/CHANGES.rst) - [Commits](https://github.com/python-telegram-bot/python-telegram-bot/compare/v12.3.0...v12.4.1) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index e1ae4a5bd..f641dd2ad 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -2,7 +2,7 @@ # mainly used for Raspberry pi installs ccxt==1.22.39 SQLAlchemy==1.3.13 -python-telegram-bot==12.3.0 +python-telegram-bot==12.4.1 arrow==0.15.5 cachetools==4.0.0 requests==2.22.0 From 83644ce5d8502ddc99c5d24a46a33750cf7745bf Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 10 Feb 2020 10:35:48 +0100 Subject: [PATCH 194/317] Fix mypy type errors in tests --- tests/data/test_history.py | 6 +++--- tests/optimize/__init__.py | 2 +- tests/optimize/test_backtesting.py | 4 ++-- tests/optimize/test_edge_cli.py | 4 ++-- tests/optimize/test_hyperopt.py | 22 ++++++++++++---------- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 7b3143db9..da4c90191 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -370,7 +370,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: def test_init(default_conf, mocker) -> None: assert {} == load_data( - datadir='', + datadir=Path(''), pairs=[], timeframe=default_conf['ticker_interval'] ) @@ -379,13 +379,13 @@ def test_init(default_conf, mocker) -> None: def test_init_with_refresh(default_conf, mocker) -> None: exchange = get_patched_exchange(mocker, default_conf) refresh_data( - datadir='', + datadir=Path(''), pairs=[], timeframe=default_conf['ticker_interval'], exchange=exchange ) assert {} == load_data( - datadir='', + datadir=Path(''), pairs=[], timeframe=default_conf['ticker_interval'] ) diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index 8756143a0..524db093e 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -23,7 +23,7 @@ class BTContainer(NamedTuple): """ Minimal BacktestContainer defining Backtest inputs and results. """ - data: List[float] + data: List[List[float]] stop_loss: float roi: Dict[str, float] trades: List[BTrade] diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 07872da57..ec85c8030 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -287,8 +287,8 @@ def test_start(mocker, fee, default_conf, caplog) -> None: '--config', 'config.json', '--strategy', 'DefaultStrategy', ] - args = get_args(args) - start_backtesting(args) + pargs = get_args(args) + start_backtesting(pargs) assert log_has('Starting freqtrade in Backtesting mode', caplog) assert start_mock.call_count == 1 diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index 96dd0899d..a5e468542 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -82,8 +82,8 @@ def test_start(mocker, fee, edge_conf, caplog) -> None: '--config', 'config.json', '--strategy', 'DefaultStrategy', ] - args = get_args(args) - start_edge(args) + pargs = get_args(args) + start_edge(pargs) assert log_has('Starting freqtrade in Edge mode', caplog) assert start_mock.call_count == 1 diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index b3356bd6d..1780b5155 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -2,6 +2,7 @@ import locale from datetime import datetime from pathlib import Path +from typing import Dict, List from unittest.mock import MagicMock, PropertyMock import pandas as pd @@ -9,7 +10,8 @@ import pytest from arrow import Arrow from filelock import Timeout -from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt +from freqtrade.commands.optimize_commands import (setup_optimize_configuration, + start_hyperopt) from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.exceptions import OperationalException @@ -54,7 +56,7 @@ def hyperopt_results(): # Functions for recurrent object patching -def create_trials(mocker, hyperopt, testdatadir) -> None: +def create_trials(mocker, hyperopt, testdatadir) -> List[Dict]: """ When creating trials, mock the hyperopt Trials so that *by default* - we don't create any pickle'd files in the filesystem @@ -228,10 +230,10 @@ def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None '--hyperopt', 'DefaultHyperOpt', '--epochs', '5' ] - args = get_args(args) + pargs = get_args(args) with pytest.raises(OperationalException, match=r"Please ensure that the hyperopt dependencies"): - start_hyperopt(args) + start_hyperopt(pargs) def test_start(mocker, default_conf, caplog) -> None: @@ -246,8 +248,8 @@ def test_start(mocker, default_conf, caplog) -> None: '--hyperopt', 'DefaultHyperOpt', '--epochs', '5' ] - args = get_args(args) - start_hyperopt(args) + pargs = get_args(args) + start_hyperopt(pargs) assert log_has('Starting freqtrade in Hyperopt mode', caplog) assert start_mock.call_count == 1 @@ -269,9 +271,9 @@ def test_start_no_data(mocker, default_conf, caplog) -> None: '--hyperopt', 'DefaultHyperOpt', '--epochs', '5' ] - args = get_args(args) + pargs = get_args(args) with pytest.raises(OperationalException, match='No data found. Terminating.'): - start_hyperopt(args) + start_hyperopt(pargs) def test_start_filelock(mocker, default_conf, caplog) -> None: @@ -286,8 +288,8 @@ def test_start_filelock(mocker, default_conf, caplog) -> None: '--hyperopt', 'DefaultHyperOpt', '--epochs', '5' ] - args = get_args(args) - start_hyperopt(args) + pargs = get_args(args) + start_hyperopt(pargs) assert log_has("Another running instance of freqtrade Hyperopt detected.", caplog) From 7bb02d0cc60074fd668a8e498c81f167fa74198e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 10 Feb 2020 11:01:33 +0100 Subject: [PATCH 195/317] Update docker-docs wording --- docs/docker.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/docker.md b/docs/docker.md index b1eb0b298..6267c0cf2 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -27,8 +27,8 @@ Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.co Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory. ``` bash -mkdir freqtrade -cd freqtrade/ +mkdir ft_userdata +cd ft_userdata/ # Download the docker-compose file from the repository curl https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docker-compose.yml -o docker-compose.yml @@ -42,15 +42,22 @@ docker-compose run --rm freqtrade create-userdir --userdir user_data docker-compose run --rm freqtrade new-config --config user_data/config.json ``` -The above snippet will create a directory called "freqtrade" - download the latest compose file and pull the freqtrade image. -The last 2 steps will create the user-directory, as well as a default configuration based on your selections. +The above snippet creates a new directory called "ft_userdata", downloads the latest compose file and pulls the freqtrade image. +The last 2 steps in the snippet create the directory with user-data, as well as (interactively) the default configuration based on your selections. + +!!! Note + You can edit the configuration at any time, which is available as `user_data/config.json` (within the directory `ft_userdata`) when using the above configuration. #### Adding your strategy The configuration is now available as `user_data/config.json`. -You should now copy your strategy to `user_data/strategies/` - and add the Strategy class name to the `docker-compose.yml` file, replacing `SampleStrategy`. +You should now copy your strategy to `user_data/strategies/` - and add the Strategy class name to the `docker-compose.yml` file, replacing `SampleStrategy`. If you wish to run the bot with the SampleStrategy, just leave it as it is. -Once this is done, you're ready to launch the bot in trading mode. +!!! Warning + The `SampleStrategy` is there for your reference and give you ideas for your own strategy. + Please always backtest the strategy and use dry-run for some time before risking real money! + +Once this is done, you're ready to launch the bot in trading mode (Dry-run or Live-trading, depending on your answer to the corresponding question you made above). ``` bash docker-compose up -d @@ -88,11 +95,11 @@ Advanced users may edit the docker-compose file further to include all possible All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. !!! Note "`docker-compose run --rm`" - Inluding `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (`freqtrade trade`). + Including `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). ##### Example: Download data with docker-compose -Downloading backtest data for one pair from binance. The data will be stored in the host directory `user_data/data/`. +Download backtesting data for 5 days for the pair ETH/BTC and 1h timeframe from Binance. The data will be stored in the directory `user_data/data/` on the host. ``` bash docker-compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h @@ -102,7 +109,7 @@ Head over to the [Data Downloading Documentation](data-download.md) for more det ##### Example: Backtest with docker-compose -Backtesting in docker-containers: +Run backtesting in docker-containers for SampleStrategy and specified timerange of historical data, on 5m timeframe: ``` bash docker-compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m @@ -126,7 +133,7 @@ You'll then also need to modify the `docker-compose.yml` file and uncomment the You can then run `docker-compose build` to build the docker image, and run it using the commands described above. -## Docker - without docker compose +## Freqtrade with docker without docker-compose !!! Warning The below documentation is provided for completeness and assumes that you are somewhat familiar with running docker containers. If you're just starting out with docker, we recommend to follow the [Freqtrade with docker-compose](#freqtrade-with-docker-compose) instructions. From d69ddd2ac37b18251b55f6f6e40995b1fac04402 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Mon, 10 Feb 2020 11:54:12 +0100 Subject: [PATCH 196/317] Apply suggestions from code review Committed 1 code suggestion in code review. Co-Authored-By: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> --- docs/telegram-usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index ac9cea3d6..c8ded4af5 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -55,7 +55,7 @@ official commands. You can ask at any moment for help with `/help`. | `/reload_conf` | | Reloads the configuration file | `/show_config` | | Shows part of the current configuration with relevant settings to operation | `/status` | | Lists all open trades -| `/status table` | | List all open trades in a table format. Pending buy orders are marked with an asterisk(*) +| `/status table` | | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) | `/count` | | Displays number of trades used and available | `/profit` | | Display a summary of your profit/loss from close trades and some stats about your performance | `/forcesell ` | | Instantly sells the given trade (Ignoring `minimum_roi`). From 0ac0ca74b5fe5db2e23d35bef0ad39174aaaabfd Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Mon, 10 Feb 2020 15:41:09 +0300 Subject: [PATCH 197/317] return back hint for running mypy --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a4a1a29f8..d84c743c9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,7 +61,7 @@ Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using #### Run mypy ``` bash -mypy freqtrade tests +mypy freqtrade ``` ## (Core)-Committer Guide From d07c69809da128471d664f2e19cc7cb28504ca5b Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 10 Feb 2020 18:32:41 +0300 Subject: [PATCH 198/317] Fix tests for hyperopt_loss --- tests/optimize/test_hyperopt.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 1780b5155..a4704b793 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -295,9 +295,12 @@ def test_start_filelock(mocker, default_conf, caplog) -> None: def test_loss_calculation_prefer_correct_trade_count(default_conf, hyperopt_results) -> None: hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, 600) - over = hl.hyperopt_loss_function(hyperopt_results, 600 + 100) - under = hl.hyperopt_loss_function(hyperopt_results, 600 - 100) + correct = hl.hyperopt_loss_function(hyperopt_results, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(hyperopt_results, 600 + 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(hyperopt_results, 600 - 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) assert over > correct assert under > correct @@ -307,8 +310,10 @@ def test_loss_calculation_prefer_shorter_trades(default_conf, hyperopt_results) resultsb.loc[1, 'trade_duration'] = 20 hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - longer = hl.hyperopt_loss_function(hyperopt_results, 100) - shorter = hl.hyperopt_loss_function(resultsb, 100) + longer = hl.hyperopt_loss_function(hyperopt_results, 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + shorter = hl.hyperopt_loss_function(resultsb, 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) assert shorter < longer @@ -319,9 +324,12 @@ def test_loss_calculation_has_limited_profit(default_conf, hyperopt_results) -> results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, 600) - over = hl.hyperopt_loss_function(results_over, 600) - under = hl.hyperopt_loss_function(results_under, 600) + correct = hl.hyperopt_loss_function(hyperopt_results, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) assert over < correct assert under > correct From faf19eda86e3728905cf94644d47275af5cabbc5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 10 Feb 2020 17:27:47 +0100 Subject: [PATCH 199/317] Break the old binary file so users are forced to reinstall Note: This should not be relevant anymore - this binary has been deprecated and is not being used by new installations since July 2019. --- bin/freqtrade | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/freqtrade b/bin/freqtrade index 25c94fe98..eee7cbef4 100755 --- a/bin/freqtrade +++ b/bin/freqtrade @@ -1,11 +1,11 @@ #!/usr/bin/env python3 import sys -import warnings +import logging -from freqtrade.main import main +logger = logging.getLogger(__name__) -warnings.warn( - "Deprecated - To continue to run the bot like this, please run `pip install -e .` again.", - DeprecationWarning) -main(sys.argv[1:]) + +logger.error("DEPRECATED installation detected, please run `pip install -e .` again.") + +sys.exit(2) From 05128d21a8be65a71d1a8a30e973b3bb26bf3884 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Mon, 10 Feb 2020 20:48:49 +0300 Subject: [PATCH 200/317] Suggest to run flake for scripts --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d84c743c9..1c83437f6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ pytest tests/test_.py::test_ #### Run Flake8 ```bash -flake8 freqtrade tests +flake8 freqtrade tests scripts ``` We receive a lot of code that fails the `flake8` checks. From c924e4d519253a400218d95b7f17a64da27e5da6 Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Mon, 10 Feb 2020 20:54:31 +0100 Subject: [PATCH 201/317] Updated based on feedback: - Profit commands now use float - Compatible with --best - Corrected wrong information in docs --- docs/utils.md | 134 ++++++++++++++++++------ freqtrade/commands/cli_options.py | 8 +- freqtrade/commands/hyperopt_commands.py | 68 ++++++------ 3 files changed, 140 insertions(+), 70 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 71039f174..5bb3a0e53 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -36,6 +36,38 @@ optional arguments: └── sample_strategy.py ``` +## Create new config + +Creates a new configuration file, asking some questions which are important selections for a configuration. + +``` +usage: freqtrade new-config [-h] [-c PATH] + +optional arguments: + -h, --help show this help message and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` + to read config from stdin. +``` + +!!! Warning + Only vital questions are asked. Freqtrade offers a lot more configuration possibilities, which are listed in the [Configuration documentation](configuration.md#configuration-parameters) + +### Create config examples + +``` +$ freqtrade new-config --config config_binance.json + +? Do you want to enable Dry-run (simulated trades)? Yes +? Please insert your stake currency: BTC +? Please insert your stake amount: 0.05 +? Please insert max_open_trades (Integer or 'unlimited'): 5 +? Please insert your ticker interval: 15m +? Please insert your display Currency (for reporting): USD +? Select exchange binance +? Do you want to enable Telegram? No +``` + ## Create new strategy Creates a new strategy from a template similar to SampleStrategy. @@ -135,27 +167,15 @@ Common arguments: ``` ``` freqtrade list-hyperopts --help -usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] [--best] - [--profitable] [--min-avg-time INT] - [--max-avg-time INT] [--min-avg-profit FLOAT] - [--min-total-profit FLOAT] [--no-color] - [--print-json] [--no-details] +usage: freqtrade list-hyperopts [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [--hyperopt-path PATH] [-1] optional arguments: -h, --help show this help message and exit - --best Select only best epochs. - --profitable Select only profitable epochs. - --min-avg-time INT Select epochs on above average time. - --max-avg-time INT Select epochs on under average time. - --min-avg-profit FLOAT - Select epochs on above average profit. - --min-total-profit FLOAT - Select epochs on above total profit. - --no-color Disable colorization of hyperopt results. May be - useful if you are redirecting output to a file. - --print-json Print best result detailization in JSON format. - --no-details Do not print best epoch details. + --hyperopt-path PATH Specify additional lookup path for Hyperopt and + Hyperopt Loss functions. + -1, --one-column Print output in one column. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -232,20 +252,31 @@ All exchanges supported by the ccxt library: _1btcxe, acx, adara, allcoin, anxpr Use the `list-timeframes` subcommand to see the list of ticker intervals (timeframes) available for the exchange. ``` -usage: freqtrade list-timeframes [-h] [--exchange EXCHANGE] [-1] +usage: freqtrade list-timeframes [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [-1] optional arguments: - -h, --help show this help message and exit - --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no - config is provided. - -1, --one-column Print output in one column. + -h, --help show this help message and exit + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. + -1, --one-column Print output in one column. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` + to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. ``` * Example: see the timeframes for the 'binance' exchange, set in the configuration file: ``` -$ freqtrade -c config_binance.json list-timeframes +$ freqtrade list-timeframes -c config_binance.json ... Timeframes available for the exchange `binance`: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M ``` @@ -269,14 +300,16 @@ You can print info about any pair/market with these subcommands - and you can fi These subcommands have same usage and same set of available options: ``` -usage: freqtrade list-markets [-h] [--exchange EXCHANGE] [--print-list] - [--print-json] [-1] [--print-csv] +usage: freqtrade list-markets [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [--exchange EXCHANGE] + [--print-list] [--print-json] [-1] [--print-csv] [--base BASE_CURRENCY [BASE_CURRENCY ...]] [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a] -usage: freqtrade list-pairs [-h] [--exchange EXCHANGE] [--print-list] - [--print-json] [-1] [--print-csv] +usage: freqtrade list-pairs [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [--exchange EXCHANGE] + [--print-list] [--print-json] [-1] [--print-csv] [--base BASE_CURRENCY [BASE_CURRENCY ...]] [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a] @@ -295,6 +328,22 @@ optional arguments: Specify quote currency(-ies). Space-separated list. -a, --all Print all pairs or market symbols. By default only active ones are shown. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + ``` By default, only active pairs/markets are shown. Active pairs/markets are those that can currently be traded @@ -316,7 +365,7 @@ $ freqtrade list-pairs --quote USD --print-json human-readable list with summary: ``` -$ freqtrade -c config_binance.json list-pairs --all --base BTC ETH --quote USDT USD --print-list +$ freqtrade list-pairs -c config_binance.json --all --base BTC ETH --quote USDT USD --print-list ``` * Print all markets on exchange "Kraken", in the tabular format: @@ -364,17 +413,40 @@ You can list the hyperoptimization epochs the Hyperopt module evaluated previous ``` usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--best] - [--profitable] [--no-color] [--print-json] - [--no-details] + [--profitable] [--min-avg-time FLOAT] + [--max-avg-time FLOAT] [--min-avg-profit FLOAT] + [--min-total-profit FLOAT] [--no-color] + [--print-json] [--no-details] optional arguments: -h, --help show this help message and exit --best Select only best epochs. --profitable Select only profitable epochs. + --min-avg-time FLOAT Select epochs on above average time. + --max-avg-time FLOAT Select epochs on under average time. + --min-avg-profit FLOAT + Select epochs on above average profit. + --min-total-profit FLOAT + Select epochs on above total profit. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. --print-json Print best result detailization in JSON format. --no-details Do not print best epoch details. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. ``` ### Examples diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 0c6d64691..154404821 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -401,14 +401,14 @@ AVAILABLE_CLI_OPTIONS = { "hyperopt_list_min_avg_time": Arg( '--min-avg-time', help='Select epochs on above average time.', - type=check_int_nonzero, - metavar='INT', + type=float, + metavar='FLOAT', ), "hyperopt_list_max_avg_time": Arg( '--max-avg-time', help='Select epochs on under average time.', - type=check_int_nonzero, - metavar='INT', + type=float, + metavar='FLOAT', ), "hyperopt_list_min_avg_profit": Arg( '--min-avg-profit', diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index cdfdb5ca6..ed0728bf6 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -27,10 +27,10 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: filteroptions = { 'only_best': config.get('hyperopt_list_best', False), 'only_profitable': config.get('hyperopt_list_profitable', False), - 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', 0), - 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', 0), - 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', 0.0), - 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', 0.0) + 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), + 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), + 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), + 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None) } trials_file = (config['user_data_dir'] / @@ -74,10 +74,10 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: filteroptions = { 'only_best': config.get('hyperopt_list_best', False), 'only_profitable': config.get('hyperopt_list_profitable', False), - 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', 0), - 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', 0), - 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', 0), - 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', 0) + 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), + 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), + 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), + 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None) } no_header = config.get('hyperopt_show_no_header', False) @@ -119,33 +119,31 @@ def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: trials = [x for x in trials if x['is_best']] if filteroptions['only_profitable']: trials = [x for x in trials if x['results_metrics']['profit'] > 0] - - if not filteroptions['only_best']: - if filteroptions['filter_min_avg_time'] > 0: - trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] - trials = [ - x for x in trials - if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] - ] - if filteroptions['filter_max_avg_time'] > 0: - trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] - trials = [ - x for x in trials - if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] - ] - if filteroptions['filter_min_avg_profit'] > 0: - trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] - trials = [ - x for x in trials - if x['results_metrics']['avg_profit'] - > filteroptions['filter_min_avg_profit'] - ] - if filteroptions['filter_min_total_profit'] > 0: - trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] - trials = [ - x for x in trials - if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] - ] + if filteroptions['filter_min_avg_time'] is not None: + trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] + trials = [ + x for x in trials + if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] + ] + if filteroptions['filter_max_avg_time'] is not None: + trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] + trials = [ + x for x in trials + if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] + ] + if filteroptions['filter_min_avg_profit'] is not None: + trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] + trials = [ + x for x in trials + if x['results_metrics']['avg_profit'] + > filteroptions['filter_min_avg_profit'] + ] + if filteroptions['filter_min_total_profit'] is not None: + trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] + trials = [ + x for x in trials + if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] + ] logger.info(f"{len(trials)} " + ("best " if filteroptions['only_best'] else "") + From f2520c11e70c3f0717bdd842f7b7cdeb5482ab0a Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Mon, 10 Feb 2020 21:19:25 +0100 Subject: [PATCH 202/317] Used wrong utils.md as base --- docs/utils.md | 83 +++++++-------------------------------------------- 1 file changed, 11 insertions(+), 72 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 5bb3a0e53..4bb2fdafb 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -36,38 +36,6 @@ optional arguments: └── sample_strategy.py ``` -## Create new config - -Creates a new configuration file, asking some questions which are important selections for a configuration. - -``` -usage: freqtrade new-config [-h] [-c PATH] - -optional arguments: - -h, --help show this help message and exit - -c PATH, --config PATH - Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` - to read config from stdin. -``` - -!!! Warning - Only vital questions are asked. Freqtrade offers a lot more configuration possibilities, which are listed in the [Configuration documentation](configuration.md#configuration-parameters) - -### Create config examples - -``` -$ freqtrade new-config --config config_binance.json - -? Do you want to enable Dry-run (simulated trades)? Yes -? Please insert your stake currency: BTC -? Please insert your stake amount: 0.05 -? Please insert max_open_trades (Integer or 'unlimited'): 5 -? Please insert your ticker interval: 15m -? Please insert your display Currency (for reporting): USD -? Select exchange binance -? Do you want to enable Telegram? No -``` - ## Create new strategy Creates a new strategy from a template similar to SampleStrategy. @@ -252,31 +220,20 @@ All exchanges supported by the ccxt library: _1btcxe, acx, adara, allcoin, anxpr Use the `list-timeframes` subcommand to see the list of ticker intervals (timeframes) available for the exchange. ``` -usage: freqtrade list-timeframes [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [-1] +usage: freqtrade list-timeframes [-h] [--exchange EXCHANGE] [-1] optional arguments: - -h, --help show this help message and exit - --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. - -1, --one-column Print output in one column. - -Common arguments: - -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. - -V, --version show program's version number and exit - -c PATH, --config PATH - Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` - to read config from stdin. - -d PATH, --datadir PATH - Path to directory with historical backtesting data. - --userdir PATH, --user-data-dir PATH - Path to userdata directory. + -h, --help show this help message and exit + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no + config is provided. + -1, --one-column Print output in one column. ``` * Example: see the timeframes for the 'binance' exchange, set in the configuration file: ``` -$ freqtrade list-timeframes -c config_binance.json +$ freqtrade -c config_binance.json list-timeframes ... Timeframes available for the exchange `binance`: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M ``` @@ -300,16 +257,14 @@ You can print info about any pair/market with these subcommands - and you can fi These subcommands have same usage and same set of available options: ``` -usage: freqtrade list-markets [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] [--exchange EXCHANGE] - [--print-list] [--print-json] [-1] [--print-csv] +usage: freqtrade list-markets [-h] [--exchange EXCHANGE] [--print-list] + [--print-json] [-1] [--print-csv] [--base BASE_CURRENCY [BASE_CURRENCY ...]] [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a] -usage: freqtrade list-pairs [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] [--exchange EXCHANGE] - [--print-list] [--print-json] [-1] [--print-csv] +usage: freqtrade list-pairs [-h] [--exchange EXCHANGE] [--print-list] + [--print-json] [-1] [--print-csv] [--base BASE_CURRENCY [BASE_CURRENCY ...]] [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a] @@ -328,22 +283,6 @@ optional arguments: Specify quote currency(-ies). Space-separated list. -a, --all Print all pairs or market symbols. By default only active ones are shown. - -Common arguments: - -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. Special values are: - 'syslog', 'journald'. See the documentation for more - details. - -V, --version show program's version number and exit - -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. - -d PATH, --datadir PATH - Path to directory with historical backtesting data. - --userdir PATH, --user-data-dir PATH - Path to userdata directory. - ``` By default, only active pairs/markets are shown. Active pairs/markets are those that can currently be traded @@ -365,7 +304,7 @@ $ freqtrade list-pairs --quote USD --print-json human-readable list with summary: ``` -$ freqtrade list-pairs -c config_binance.json --all --base BTC ETH --quote USDT USD --print-list +$ freqtrade -c config_binance.json list-pairs --all --base BTC ETH --quote USDT USD --print-list ``` * Print all markets on exchange "Kraken", in the tabular format: From 62bcb3d7660ad60f0b9c3f7374878df16117c253 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 11 Feb 2020 03:43:20 +0300 Subject: [PATCH 203/317] Fix tests in test_history.py --- tests/data/test_history.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index da4c90191..15f507b90 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -420,7 +420,7 @@ def test_trim_tickerlist(testdatadir) -> None: # Test the pattern ^(\d{8})-$ # This pattern extracts elements from the date to now - timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, None) + timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, 0) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -430,14 +430,14 @@ def test_trim_tickerlist(testdatadir) -> None: # Test a wrong pattern # This pattern must return the list unchanged - timerange = TimeRange(None, None, None, 5) + timerange = TimeRange(None, None, 0, 5) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) assert ticker_list_len == ticker_len # passing empty list - timerange = TimeRange(None, None, None, 5) + timerange = TimeRange(None, None, 0, 5) ticker = trim_tickerlist([], timerange) assert 0 == len(ticker) assert not ticker From 29f7c5071b2c99536f1499ba420f7c09a7eadcf2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 11 Feb 2020 04:17:10 +0300 Subject: [PATCH 204/317] Fix usage of an item from BTContainer in tests --- tests/optimize/__init__.py | 4 ++-- tests/optimize/test_backtest_detail.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index 524db093e..13605a38c 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -1,4 +1,4 @@ -from typing import Dict, List, NamedTuple +from typing import Dict, List, NamedTuple, Optional import arrow from pandas import DataFrame @@ -30,7 +30,7 @@ class BTContainer(NamedTuple): profit_perc: float trailing_stop: bool = False trailing_only_offset_is_reached: bool = False - trailing_stop_positive: float = None + trailing_stop_positive: Optional[float] = None trailing_stop_positive_offset: float = 0.0 use_sell_signal: bool = False diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index bd2765430..e7bc76c1d 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -364,7 +364,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: default_conf["trailing_stop"] = data.trailing_stop default_conf["trailing_only_offset_is_reached"] = data.trailing_only_offset_is_reached # Only add this to configuration If it's necessary - if data.trailing_stop_positive: + if data.trailing_stop_positive is not None: default_conf["trailing_stop_positive"] = data.trailing_stop_positive default_conf["trailing_stop_positive_offset"] = data.trailing_stop_positive_offset default_conf["ask_strategy"] = {"use_sell_signal": data.use_sell_signal} From f99d1c38298b717a3e1d1299477338e41746c756 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 11 Feb 2020 15:44:47 +0100 Subject: [PATCH 205/317] fixed open_rate instead of open_rate_requested --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5f1024f8c..aa617a386 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -549,7 +549,7 @@ class FreqtradeBot: 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, - 'limit': trade.open_rate_requested, + 'limit': trade.open_rate, 'order_type': order_type, 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], From 7f4b90c68f9fd1a116ba3179b335add64c66c3cd Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 11 Feb 2020 15:45:35 +0100 Subject: [PATCH 206/317] fixed actual open_rate in notify_buy_cancel --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index aa617a386..6d1122aa1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -572,7 +572,7 @@ class FreqtradeBot: 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, - 'limit': trade.open_rate_requested, + 'limit': trade.open_rate, 'order_type': order_type, 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], From 867b736b8477507f933ce8b9768a433de5721615 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 11 Feb 2020 15:49:57 +0100 Subject: [PATCH 207/317] Fixed to Executing Buys & Sells --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6d1122aa1..85ddb0da1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -414,7 +414,7 @@ class FreqtradeBot: if ((bid_check_dom.get('enabled', False)) and (bid_check_dom.get('bids_to_ask_delta', 0) > 0)): if self._check_depth_of_market_buy(pair, bid_check_dom): - logger.info(f'Executed Buy for {pair}.') + logger.info(f'Executing Buy for {pair}.') return self.execute_buy(pair, stake_amount) else: return False @@ -804,8 +804,8 @@ class FreqtradeBot: ) if should_sell.sell_flag: + logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}') self.execute_sell(trade, sell_rate, should_sell.sell_type) - logger.info(f'Executed Sell for {trade.pair}. Reason: {should_sell.sell_type}') return True return False From fc29564974d77e54599bf759dd1d1ed4d2df884a Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 11 Feb 2020 15:58:40 +0100 Subject: [PATCH 208/317] Fixed messages and readability --- docs/webhook-config.md | 4 ++-- freqtrade/rpc/rpc.py | 4 ++-- freqtrade/rpc/telegram.py | 4 ++-- tests/rpc/test_rpc_telegram.py | 6 +++--- tests/rpc/test_rpc_webhook.py | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/webhook-config.md b/docs/webhook-config.md index b287fa71e..e53aa8af5 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -16,7 +16,7 @@ Sample configuration (tested using IFTTT). "value3": "{stake_amount:8f} {stake_currency}" }, "webhookbuycancel": { - "value1": "Cancelling Buy {pair}", + "value1": "Cancelling Open Buy Order for {pair}", "value2": "limit {limit:8f}", "value3": "{stake_amount:8f} {stake_currency}" }, @@ -26,7 +26,7 @@ Sample configuration (tested using IFTTT). "value3": "profit: {profit_amount:8f} {stake_currency}" }, "webhooksellcancel": { - "value1": "Cancelling Sell {pair}", + "value1": "Cancelling Open Sell Order for {pair}", "value2": "limit {limit:8f}", "value3": "profit: {profit_amount:8f} {stake_currency}" }, diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c1efea79e..07631f258 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -168,8 +168,8 @@ class RPC: profit_str += f" ({fiat_profit:.2f})" trades_list.append([ trade.id, - trade.pair + ['', '*'][trade.open_order_id is not None - and trade.close_rate_requested is None], + trade.pair + '*' if (trade.open_order_id is not None + and trade.close_rate_requested is None) else '', shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), profit_str ]) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0dd7a8ffd..e3d4f54e7 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -144,7 +144,7 @@ class Telegram(RPC): message += ")`" elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION: - message = "*{exchange}:* Cancelling Buy {pair}".format(**msg) + message = "*{exchange}:* Cancelling Open Buy Order for {pair}".format(**msg) elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: msg['amount'] = round(msg['amount'], 8) @@ -172,7 +172,7 @@ class Telegram(RPC): ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg) elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION: - message = "*{exchange}:* Cancelling Sell {pair}".format(**msg) + message = "*{exchange}:* Cancelling Open Sell Order for {pair}".format(**msg) elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: message = '*Status:* `{status}`'.format(**msg) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index ae9c0c4dc..a8b8e0c5a 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1228,7 +1228,7 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: 'pair': 'ETH/BTC', }) assert msg_mock.call_args[0][0] \ - == ('*Bittrex:* Cancelling Buy ETH/BTC') + == ('*Bittrex:* Cancelling Open Buy Order for ETH/BTC') def test_send_msg_sell_notification(default_conf, mocker) -> None: @@ -1318,7 +1318,7 @@ def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None: 'pair': 'KEY/ETH', }) assert msg_mock.call_args[0][0] \ - == ('*Binance:* Cancelling Sell KEY/ETH') + == ('*Binance:* Cancelling Open Sell Order for KEY/ETH') msg_mock.reset_mock() telegram.send_msg({ @@ -1327,7 +1327,7 @@ def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None: 'pair': 'KEY/ETH', }) assert msg_mock.call_args[0][0] \ - == ('*Binance:* Cancelling Sell KEY/ETH') + == ('*Binance:* Cancelling Open Sell Order for KEY/ETH') # Reset singleton function to avoid random breaks telegram._fiat_converter.convert_amount = old_convamount diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index ab40047c0..3f3f36766 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -21,7 +21,7 @@ def get_webhook_dict() -> dict: "value3": "{stake_amount:8f} {stake_currency}" }, "webhookbuycancel": { - "value1": "Cancelling Buy {pair}", + "value1": "Cancelling Open Buy Order for {pair}", "value2": "limit {limit:8f}", "value3": "{stake_amount:8f} {stake_currency}" }, @@ -31,7 +31,7 @@ def get_webhook_dict() -> dict: "value3": "profit: {profit_amount:8f} {stake_currency}" }, "webhooksellcancel": { - "value1": "Cancelling Sell {pair}", + "value1": "Cancelling Open Sell Order for {pair}", "value2": "limit {limit:8f}", "value3": "profit: {profit_amount:8f} {stake_currency}" }, From 5b4d8d69ef567ca074351186eebe97e3c9e2fe52 Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Tue, 11 Feb 2020 16:02:08 +0100 Subject: [PATCH 209/317] Adding --min-trades and --max-trades for hyperopt-list --- docs/utils.md | 5 +++- freqtrade/commands/arguments.py | 1 + freqtrade/commands/cli_options.py | 12 ++++++++++ freqtrade/commands/hyperopt_commands.py | 14 ++++++++++++ freqtrade/configuration/configuration.py | 6 +++++ tests/commands/test_commands.py | 29 ++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 1 deletion(-) mode change 100644 => 100755 freqtrade/commands/hyperopt_commands.py diff --git a/docs/utils.md b/docs/utils.md index 5bb3a0e53..91dd6eae0 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -413,7 +413,8 @@ You can list the hyperoptimization epochs the Hyperopt module evaluated previous ``` usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--best] - [--profitable] [--min-avg-time FLOAT] + [--profitable] [--min-trades INT] + [--max-trades INT] [--min-avg-time FLOAT] [--max-avg-time FLOAT] [--min-avg-profit FLOAT] [--min-total-profit FLOAT] [--no-color] [--print-json] [--no-details] @@ -422,6 +423,8 @@ optional arguments: -h, --help show this help message and exit --best Select only best epochs. --profitable Select only profitable epochs. + --min-trades INT Select epochs with more than INT trades. + --max-trades INT Select epochs with less than INT trades. --min-avg-time FLOAT Select epochs on above average time. --max-avg-time FLOAT Select epochs on under average time. --min-avg-profit FLOAT diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index e5a68389b..1b2c4482e 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -62,6 +62,7 @@ ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "ticker_interval"] ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", + "hyperopt_list_min_trades", "hyperopt_list_max_trades", "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time", "hyperopt_list_min_avg_profit", "hyperopt_list_min_total_profit", "print_colorized", "print_json", "hyperopt_list_no_details"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 154404821..f9351c207 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -398,6 +398,18 @@ AVAILABLE_CLI_OPTIONS = { help='Select only best epochs.', action='store_true', ), + "hyperopt_list_min_trades": Arg( + '--min-trades', + help='Select epochs with more than INT trades.', + type=check_int_nonzero, + metavar='INT', + ), + "hyperopt_list_max_trades": Arg( + '--max-trades', + help='Select epochs with less than INT trades.', + type=check_int_nonzero, + metavar='INT', + ), "hyperopt_list_min_avg_time": Arg( '--min-avg-time', help='Select epochs on above average time.', diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py old mode 100644 new mode 100755 index ed0728bf6..c3baf2406 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -27,6 +27,8 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: filteroptions = { 'only_best': config.get('hyperopt_list_best', False), 'only_profitable': config.get('hyperopt_list_profitable', False), + 'filter_min_trades': config.get('hyperopt_list_min_trades', 0), + 'filter_max_trades': config.get('hyperopt_list_max_trades', 0), 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), @@ -74,6 +76,8 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: filteroptions = { 'only_best': config.get('hyperopt_list_best', False), 'only_profitable': config.get('hyperopt_list_profitable', False), + 'filter_min_trades': config.get('hyperopt_list_min_trades', 0), + 'filter_max_trades': config.get('hyperopt_list_max_trades', 0), 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), @@ -119,6 +123,16 @@ def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: trials = [x for x in trials if x['is_best']] if filteroptions['only_profitable']: trials = [x for x in trials if x['results_metrics']['profit'] > 0] + if filteroptions['filter_min_trades'] > 0: + trials = [ + x for x in trials + if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades'] + ] + if filteroptions['filter_max_trades'] > 0: + trials = [ + x for x in trials + if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] + ] if filteroptions['filter_min_avg_time'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index f7e87f3a1..41f24e55c 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -310,6 +310,12 @@ class Configuration: self._args_to_config(config, argname='hyperopt_list_profitable', logstring='Parameter --profitable detected: {}') + self._args_to_config(config, argname='hyperopt_list_min_trades', + logstring='Parameter --min-trades detected: {}') + + self._args_to_config(config, argname='hyperopt_list_max_trades', + logstring='Parameter --max-trades detected: {}') + self._args_to_config(config, argname='hyperopt_list_min_avg_time', logstring='Parameter --min-avg-time detected: {}') diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index db8a9289a..e02a721a4 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -773,6 +773,35 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): pargs['config'] = None start_hyperopt_list(pargs) captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--min-trades", "20" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 3/12", " 6/12", " 7/12", " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 4/12", " 5/12", " 8/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--max-trades", "20" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() assert all(x in captured.out for x in [" 2/12", " 10/12"]) assert all(x not in captured.out From 4fedf1e564212b43b359a1bc533f0d571b9ce926 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 11 Feb 2020 16:05:44 +0100 Subject: [PATCH 210/317] default refresh TRUE on get_buy_rate and get_sell_Rate --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 85ddb0da1..c04e3077e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -234,7 +234,7 @@ class FreqtradeBot: return trades_created - def get_buy_rate(self, pair: str, refresh: bool = False, tick: Dict = None) -> float: + def get_buy_rate(self, pair: str, refresh: bool = True, tick: Dict = None) -> float: """ Calculates bid target between current ask price and last price :return: float: Price @@ -615,7 +615,7 @@ class FreqtradeBot: return trades_closed - def get_sell_rate(self, pair: str, refresh: bool = False) -> float: + def get_sell_rate(self, pair: str, refresh: bool = True) -> float: """ Get sell rate - either using get-ticker bid or first bid based on orderbook The orderbook portion is only used for rpc messaging, which would otherwise fail From 5f4c209fca844f6ea66f7302c22a64d7375812db Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 11 Feb 2020 16:14:49 +0100 Subject: [PATCH 211/317] fixed one more occurence of executed buy, and test --- freqtrade/freqtradebot.py | 2 +- tests/test_freqtradebot.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c04e3077e..0d1105b2f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -419,7 +419,7 @@ class FreqtradeBot: else: return False - logger.info(f'Executed Buy for {pair}') + logger.info(f'Executing Buy for {pair}') return self.execute_buy(pair, stake_amount) else: return False diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 429d3599d..18cd81aed 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -300,7 +300,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf # stoploss shoud be hit assert freqtrade.handle_trade(trade) is True - assert log_has('Executed Sell for NEO/BTC. Reason: SellType.STOP_LOSS', caplog) + assert log_has('Executing Sell for NEO/BTC. Reason: SellType.STOP_LOSS', caplog) assert trade.sell_reason == SellType.STOP_LOSS.value From cde1b2b56c7fa6285af72ec3d1758968f4d9b88d Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 11 Feb 2020 16:28:48 +0100 Subject: [PATCH 212/317] readded rpc status message for partial buys --- freqtrade/freqtradebot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0d1105b2f..ffd951ee2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -906,6 +906,10 @@ class FreqtradeBot: trade.open_order_id = None logger.info('Partial buy order timeout for %s.', trade) + self.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': f'Remaining buy order for {trade.pair} cancelled due to timeout' + }) return False def handle_timedout_limit_sell(self, trade: Trade, order: Dict) -> bool: From 899de8b27c3f35273f090534b0bb2a9d131270f6 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 11 Feb 2020 16:50:18 +0100 Subject: [PATCH 213/317] modified tests for double partial call --- tests/test_freqtradebot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 18cd81aed..c0af1f015 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2067,7 +2067,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old # note this is for a partially-complete buy order freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 + assert rpc_mock.call_count == 2 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() assert len(trades) == 1 assert trades[0].amount == 23.0 @@ -2101,7 +2101,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap assert log_has_re(r"Applying fee on amount for Trade.* Order", caplog) assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 + assert rpc_mock.call_count == 2 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() assert len(trades) == 1 # Verify that tradehas been updated @@ -2140,7 +2140,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, assert log_has_re(r"Could not update trade amount: .*", caplog) assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 + assert rpc_mock.call_count == 2 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() assert len(trades) == 1 # Verify that tradehas been updated From d1c3eabb870fe5a5f2357086459b1a2ca06faaa9 Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Tue, 11 Feb 2020 18:08:30 +0100 Subject: [PATCH 214/317] Changed commands to use "check_int_positive" --- freqtrade/commands/cli_options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index f9351c207..c3b79ae3a 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -401,13 +401,13 @@ AVAILABLE_CLI_OPTIONS = { "hyperopt_list_min_trades": Arg( '--min-trades', help='Select epochs with more than INT trades.', - type=check_int_nonzero, + type=check_int_positive, metavar='INT', ), "hyperopt_list_max_trades": Arg( '--max-trades', help='Select epochs with less than INT trades.', - type=check_int_nonzero, + type=check_int_positive, metavar='INT', ), "hyperopt_list_min_avg_time": Arg( From c35fe2c386ae84a128e817be79c901a5345538c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Feb 2020 19:29:43 +0100 Subject: [PATCH 215/317] Add link to quick-start-guide --- docs/docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docker.md b/docs/docker.md index 6267c0cf2..cd24994bc 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -8,7 +8,7 @@ Start by downloading and installing Docker CE for your platform: * [Windows](https://docs.docker.com/docker-for-windows/install/) * [Linux](https://docs.docker.com/install/) -Optionally, [docker-compose](https://docs.docker.com/compose/install/) should be installed and available to follow the docker quick start guide. +Optionally, [docker-compose](https://docs.docker.com/compose/install/) should be installed and available to follow the [docker quick start guide](#docker-quick-start). Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. From 7be9f0067e63f7a889e10c096b1f3fd36cbedf81 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Feb 2020 20:45:53 +0100 Subject: [PATCH 216/317] Update data-analysis documentation to properly initialize configuration --- docs/strategy_analysis_example.md | 27 +++++++++---------- .../templates/strategy_analysis_example.ipynb | 27 +++++++++---------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index cc6b9805f..2d77edaed 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -7,18 +7,19 @@ Debugging a strategy can be time-consuming. FreqTrade offers helper functions to ```python from pathlib import Path +from freqtrade.configuration import Configuration + # Customize these according to your needs. +# Initialize empty configuration object +config = Configuration.from_files([]) +# Optionally, Use existing configuration file +# config = Configuration.from_files(["config.json"]) + # Define some constants -timeframe = "5m" +config["ticker_interval"] = "5m" # Name of the strategy class -strategy_name = 'SampleStrategy' -# Path to user data -user_data_dir = Path('user_data') -# Location of the strategy -strategy_location = user_data_dir / 'strategies' -# Location of the data -data_location = Path(user_data_dir, 'data', 'binance') +config["strategy"] = "SampleStrategy" # Pair to analyze - Only use one pair here pair = "BTC_USDT" ``` @@ -28,8 +29,8 @@ pair = "BTC_USDT" # Load data using values set above from freqtrade.data.history import load_pair_history -candles = load_pair_history(datadir=data_location, - timeframe=timeframe, +candles = load_pair_history(datadir=config["data_dir"], + timeframe=config["ticker_interval"], pair=pair) # Confirm success @@ -44,9 +45,7 @@ candles.head() ```python # Load strategy using values set above from freqtrade.resolvers import StrategyResolver -strategy = StrategyResolver.load_strategy({'strategy': strategy_name, - 'user_data_dir': user_data_dir, - 'strategy_path': strategy_location}) +strategy = StrategyResolver.load_strategy(config) # Generate buy/sell signals using strategy df = strategy.analyze_ticker(candles, {'pair': pair}) @@ -86,7 +85,7 @@ Analyze a trades dataframe (also used below for plotting) from freqtrade.data.btanalysis import load_backtest_data # Load backtest results -trades = load_backtest_data(user_data_dir / "backtest_results/backtest-result.json") +trades = load_backtest_data(config["user_data_dir"] / "backtest_results/backtest-result.json") # Show value-counts per pair trades.groupby("pair")["sell_reason"].value_counts() diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index eea8fb85f..06fc3f557 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -23,18 +23,19 @@ "outputs": [], "source": [ "from pathlib import Path\n", + "from freqtrade.configuration import Configuration\n", + "\n", "# Customize these according to your needs.\n", "\n", + "# Initialize empty configuration object\n", + "config = Configuration.from_files([])\n", + "# Optionally, Use existing configuration file\n", + "# config = Configuration.from_files([\"config.json\"])\n", + "\n", "# Define some constants\n", - "timeframe = \"5m\"\n", + "config[\"ticker_interval\"] = \"5m\"\n", "# Name of the strategy class\n", - "strategy_name = 'SampleStrategy'\n", - "# Path to user data\n", - "user_data_dir = Path('user_data')\n", - "# Location of the strategy\n", - "strategy_location = user_data_dir / 'strategies'\n", - "# Location of the data\n", - "data_location = Path(user_data_dir, 'data', 'binance')\n", + "config[\"strategy\"] = \"SampleStrategy\"\n", "# Pair to analyze - Only use one pair here\n", "pair = \"BTC_USDT\"" ] @@ -48,8 +49,8 @@ "# Load data using values set above\n", "from freqtrade.data.history import load_pair_history\n", "\n", - "candles = load_pair_history(datadir=data_location,\n", - " timeframe=timeframe,\n", + "candles = load_pair_history(datadir=config[\"data_dir\"],\n", + " timeframe=config[\"ticker_interval\"],\n", " pair=pair)\n", "\n", "# Confirm success\n", @@ -73,9 +74,7 @@ "source": [ "# Load strategy using values set above\n", "from freqtrade.resolvers import StrategyResolver\n", - "strategy = StrategyResolver.load_strategy({'strategy': strategy_name,\n", - " 'user_data_dir': user_data_dir,\n", - " 'strategy_path': strategy_location})\n", + "strategy = StrategyResolver.load_strategy(config)\n", "\n", "# Generate buy/sell signals using strategy\n", "df = strategy.analyze_ticker(candles, {'pair': pair})\n", @@ -137,7 +136,7 @@ "from freqtrade.data.btanalysis import load_backtest_data\n", "\n", "# Load backtest results\n", - "trades = load_backtest_data(user_data_dir / \"backtest_results/backtest-result.json\")\n", + "trades = load_backtest_data(config[\"user_data_dir\"] / \"backtest_results/backtest-result.json\")\n", "\n", "# Show value-counts per pair\n", "trades.groupby(\"pair\")[\"sell_reason\"].value_counts()" From 539343b20d7304e3f746c3ac7868bb584f3e1539 Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Tue, 11 Feb 2020 21:29:55 +0100 Subject: [PATCH 217/317] Adding 2 more filter options for completeness --- docs/utils.md | 8 ++++++- freqtrade/commands/arguments.py | 3 ++- freqtrade/commands/cli_options.py | 12 ++++++++++ freqtrade/commands/hyperopt_commands.py | 21 ++++++++++++++++-- freqtrade/configuration/configuration.py | 6 +++++ tests/commands/test_commands.py | 28 ++++++++++++++++++++++++ 6 files changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 91dd6eae0..abb7fd0db 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -416,7 +416,9 @@ usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--profitable] [--min-trades INT] [--max-trades INT] [--min-avg-time FLOAT] [--max-avg-time FLOAT] [--min-avg-profit FLOAT] - [--min-total-profit FLOAT] [--no-color] + [--max-avg-profit FLOAT] + [--min-total-profit FLOAT] + [--max-total-profit FLOAT] [--no-color] [--print-json] [--no-details] optional arguments: @@ -429,8 +431,12 @@ optional arguments: --max-avg-time FLOAT Select epochs on under average time. --min-avg-profit FLOAT Select epochs on above average profit. + --max-avg-profit FLOAT + Select epochs on below average profit. --min-total-profit FLOAT Select epochs on above total profit. + --max-total-profit FLOAT + Select epochs on below total profit. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. --print-json Print best result detailization in JSON format. diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 1b2c4482e..fe6f49039 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -64,7 +64,8 @@ ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_list_min_trades", "hyperopt_list_max_trades", "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time", - "hyperopt_list_min_avg_profit", "hyperopt_list_min_total_profit", + "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit", + "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit", "print_colorized", "print_json", "hyperopt_list_no_details"] ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index c3b79ae3a..1776955b1 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -428,12 +428,24 @@ AVAILABLE_CLI_OPTIONS = { type=float, metavar='FLOAT', ), + "hyperopt_list_max_avg_profit": Arg( + '--max-avg-profit', + help='Select epochs on below average profit.', + type=float, + metavar='FLOAT', + ), "hyperopt_list_min_total_profit": Arg( '--min-total-profit', help='Select epochs on above total profit.', type=float, metavar='FLOAT', ), + "hyperopt_list_max_total_profit": Arg( + '--max-total-profit', + help='Select epochs on below total profit.', + type=float, + metavar='FLOAT', + ), "hyperopt_list_no_details": Arg( '--no-details', help='Do not print best epoch details.', diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index c3baf2406..8c1c80d98 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -32,7 +32,9 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), - 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None) + 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None), + 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), + 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None) } trials_file = (config['user_data_dir'] / @@ -81,7 +83,9 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), - 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None) + 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None), + 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), + 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None) } no_header = config.get('hyperopt_show_no_header', False) @@ -152,12 +156,25 @@ def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit'] ] + if filteroptions['filter_max_avg_profit'] is not None: + trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] + trials = [ + x for x in trials + if x['results_metrics']['avg_profit'] + < filteroptions['filter_max_avg_profit'] + ] if filteroptions['filter_min_total_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ x for x in trials if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] ] + if filteroptions['filter_max_total_profit'] is not None: + trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] + trials = [ + x for x in trials + if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] + ] logger.info(f"{len(trials)} " + ("best " if filteroptions['only_best'] else "") + diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 41f24e55c..c2613ba99 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -325,9 +325,15 @@ class Configuration: self._args_to_config(config, argname='hyperopt_list_min_avg_profit', logstring='Parameter --min-avg-profit detected: {}') + self._args_to_config(config, argname='hyperopt_list_max_avg_profit', + logstring='Parameter --max-avg-profit detected: {}') + self._args_to_config(config, argname='hyperopt_list_min_total_profit', logstring='Parameter --min-total-profit detected: {}') + self._args_to_config(config, argname='hyperopt_list_max_total_profit', + logstring='Parameter --max-total-profit detected: {}') + self._args_to_config(config, argname='hyperopt_list_no_details', logstring='Parameter --no-details detected: {}') diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index e02a721a4..ee1db5db5 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -822,6 +822,20 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): assert all(x not in captured.out for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", " 10/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--max-avg-profit", "0.10" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12"]) + assert all(x not in captured.out + for x in [" 2/12", " 4/12", " 10/12", " 12/12"]) args = [ "hyperopt-list", "--no-details", @@ -836,6 +850,20 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): assert all(x not in captured.out for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--max-total-profit", "0.4" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 4/12", " 10/12", " 12/12"]) args = [ "hyperopt-list", "--profitable", From 4f3376e2a189618f649e8f5d91d3ae753b1ae730 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 12 Feb 2020 01:39:15 +0300 Subject: [PATCH 218/317] Do not instantiate directly DefaultStrategy in tests --- tests/data/test_history.py | 14 ++++++++++---- tests/optimize/test_backtesting.py | 6 ++++-- tests/strategy/test_interface.py | 18 ++++++++++++------ tests/test_plotting.py | 14 +++++++++----- 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 15f507b90..cf0901587 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -24,7 +24,7 @@ from freqtrade.data.history import (_download_pair_history, validate_backtest_data) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json -from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.resolvers import StrategyResolver from tests.conftest import (get_patched_exchange, log_has, log_has_re, patch_exchange) @@ -509,7 +509,9 @@ def test_file_dump_json_tofile(testdatadir) -> None: def test_get_timerange(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) - strategy = DefaultStrategy(default_conf) + + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) data = strategy.tickerdata_to_dataframe( load_data( @@ -525,7 +527,9 @@ def test_get_timerange(default_conf, mocker, testdatadir) -> None: def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) -> None: patch_exchange(mocker) - strategy = DefaultStrategy(default_conf) + + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) data = strategy.tickerdata_to_dataframe( load_data( @@ -547,7 +551,9 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> None: patch_exchange(mocker) - strategy = DefaultStrategy(default_conf) + + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) timerange = TimeRange('index', 'index', 200, 250) data = strategy.tickerdata_to_dataframe( diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index ec85c8030..bba15c156 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -20,8 +20,8 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.optimize.backtesting import Backtesting +from freqtrade.resolvers import StrategyResolver from freqtrade.state import RunMode -from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -350,7 +350,9 @@ def test_tickerdata_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: assert len(data['UNITTEST/BTC']) == 102 # Load strategy to compare the result between Backtesting function and strategy are the same - strategy = DefaultStrategy(default_conf) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) + data2 = strategy.tickerdata_to_dataframe(tickerlist) assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC']) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 89c38bda1..a28519383 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -10,8 +10,9 @@ from freqtrade.configuration import TimeRange from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.persistence import Trade -from tests.conftest import get_patched_exchange, log_has +from freqtrade.resolvers import StrategyResolver from freqtrade.strategy.default_strategy import DefaultStrategy +from tests.conftest import get_patched_exchange, log_has # Avoid to reinit the same object again and again _STRATEGY = DefaultStrategy(config={}) @@ -104,7 +105,8 @@ def test_get_signal_handles_exceptions(mocker, default_conf): def test_tickerdata_to_dataframe(default_conf, testdatadir) -> None: - strategy = DefaultStrategy(default_conf) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) timerange = TimeRange.parse_timerange('1510694220-1510700340') tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange) @@ -120,7 +122,8 @@ def test_min_roi_reached(default_conf, fee) -> None: min_roi_list = [{20: 0.05, 55: 0.01, 0: 0.1}, {0: 0.1, 20: 0.05, 55: 0.01}] for roi in min_roi_list: - strategy = DefaultStrategy(default_conf) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) strategy.minimal_roi = roi trade = Trade( pair='ETH/BTC', @@ -158,7 +161,8 @@ def test_min_roi_reached2(default_conf, fee) -> None: }, ] for roi in min_roi_list: - strategy = DefaultStrategy(default_conf) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) strategy.minimal_roi = roi trade = Trade( pair='ETH/BTC', @@ -192,7 +196,8 @@ def test_min_roi_reached3(default_conf, fee) -> None: 30: 0.05, 55: 0.30, } - strategy = DefaultStrategy(default_conf) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) strategy.minimal_roi = min_roi trade = Trade( pair='ETH/BTC', @@ -292,7 +297,8 @@ def test__analyze_ticker_internal_skip_analyze(ticker_history, mocker, caplog) - def test_is_pair_locked(default_conf): - strategy = DefaultStrategy(default_conf) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) # dict should be empty assert not strategy._pair_locked_until diff --git a/tests/test_plotting.py b/tests/test_plotting.py index e7ec4ce46..34d1f2b0c 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -19,7 +19,7 @@ from freqtrade.plot.plotting import (add_indicators, add_profit, generate_profit_graph, init_plotscript, load_and_plot_trades, plot_profit, plot_trades, store_plot_file) -from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.resolvers import StrategyResolver from tests.conftest import get_args, log_has, log_has_re @@ -70,9 +70,11 @@ def test_add_indicators(default_conf, testdatadir, caplog): indicators1 = {"ema10": {}} indicators2 = {"macd": {"color": "red"}} + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) + # Generate buy/sell signals and indicators - strat = DefaultStrategy(default_conf) - data = strat.analyze_ticker(data, {'pair': pair}) + data = strategy.analyze_ticker(data, {'pair': pair}) fig = generate_empty_figure() # Row 1 @@ -181,9 +183,11 @@ def test_generate_candlestick_graph_no_trades(default_conf, mocker, testdatadir) data = history.load_pair_history(pair=pair, timeframe='1m', datadir=testdatadir, timerange=timerange) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) + # Generate buy/sell signals and indicators - strat = DefaultStrategy(default_conf) - data = strat.analyze_ticker(data, {'pair': pair}) + data = strategy.analyze_ticker(data, {'pair': pair}) indicators1 = [] indicators2 = [] From d6b9397579c2d4542bf5742a9ac7aa7f68a56f08 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Feb 2020 06:40:13 +0100 Subject: [PATCH 219/317] Fix typo in datadir key --- docs/strategy_analysis_example.md | 4 ++-- freqtrade/templates/strategy_analysis_example.ipynb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 2d77edaed..10ae73450 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -13,7 +13,7 @@ from freqtrade.configuration import Configuration # Initialize empty configuration object config = Configuration.from_files([]) -# Optionally, Use existing configuration file +# Optionally, use existing configuration file # config = Configuration.from_files(["config.json"]) # Define some constants @@ -29,7 +29,7 @@ pair = "BTC_USDT" # Load data using values set above from freqtrade.data.history import load_pair_history -candles = load_pair_history(datadir=config["data_dir"], +candles = load_pair_history(datadir=config["datadir"], timeframe=config["ticker_interval"], pair=pair) diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 06fc3f557..4541f5ed5 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -29,7 +29,7 @@ "\n", "# Initialize empty configuration object\n", "config = Configuration.from_files([])\n", - "# Optionally, Use existing configuration file\n", + "# Optionally, use existing configuration file\n", "# config = Configuration.from_files([\"config.json\"])\n", "\n", "# Define some constants\n", @@ -49,7 +49,7 @@ "# Load data using values set above\n", "from freqtrade.data.history import load_pair_history\n", "\n", - "candles = load_pair_history(datadir=config[\"data_dir\"],\n", + "candles = load_pair_history(datadir=config[\"datadir\"],\n", " timeframe=config[\"ticker_interval\"],\n", " pair=pair)\n", "\n", From 483cba453a3f9961ed6905b780de1a72881bdd20 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Feb 2020 19:58:23 +0100 Subject: [PATCH 220/317] Fix last occurence of data_location --- docs/strategy_analysis_example.md | 2 +- freqtrade/templates/strategy_analysis_example.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 10ae73450..97a555e9e 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -34,7 +34,7 @@ candles = load_pair_history(datadir=config["datadir"], pair=pair) # Confirm success -print("Loaded " + str(len(candles)) + f" rows of data for {pair} from {data_location}") +print("Loaded " + str(len(candles)) + f" rows of data for {pair} from {config['datadir']}") candles.head() ``` diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 4541f5ed5..4b904e100 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -54,7 +54,7 @@ " pair=pair)\n", "\n", "# Confirm success\n", - "print(\"Loaded \" + str(len(candles)) + f\" rows of data for {pair} from {data_location}\")\n", + "print(\"Loaded \" + str(len(candles)) + f\" rows of data for {pair} from {config['datadir']}\")\n", "candles.head()" ] }, From 2efa1c164fa4c1b0f5bab1775fba7b0c496ef299 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Feb 2020 21:43:43 +0100 Subject: [PATCH 221/317] Revert data-location section --- docs/strategy_analysis_example.md | 6 ++++-- freqtrade/templates/strategy_analysis_example.ipynb | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 97a555e9e..93e84122b 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -20,6 +20,8 @@ config = Configuration.from_files([]) config["ticker_interval"] = "5m" # Name of the strategy class config["strategy"] = "SampleStrategy" +# Location of the data +data_location = Path(config['user_data_dir'], 'data', 'binance') # Pair to analyze - Only use one pair here pair = "BTC_USDT" ``` @@ -29,12 +31,12 @@ pair = "BTC_USDT" # Load data using values set above from freqtrade.data.history import load_pair_history -candles = load_pair_history(datadir=config["datadir"], +candles = load_pair_history(datadir=data_location, timeframe=config["ticker_interval"], pair=pair) # Confirm success -print("Loaded " + str(len(candles)) + f" rows of data for {pair} from {config['datadir']}") +print("Loaded " + str(len(candles)) + f" rows of data for {pair} from {data_location}") candles.head() ``` diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 4b904e100..91e132380 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -36,6 +36,8 @@ "config[\"ticker_interval\"] = \"5m\"\n", "# Name of the strategy class\n", "config[\"strategy\"] = \"SampleStrategy\"\n", + "# Location of the data\n", + "data_location = Path(config['user_data_dir'], 'data', 'binance')\n", "# Pair to analyze - Only use one pair here\n", "pair = \"BTC_USDT\"" ] @@ -49,12 +51,12 @@ "# Load data using values set above\n", "from freqtrade.data.history import load_pair_history\n", "\n", - "candles = load_pair_history(datadir=config[\"datadir\"],\n", + "candles = load_pair_history(datadir=data_location,\n", " timeframe=config[\"ticker_interval\"],\n", " pair=pair)\n", "\n", "# Confirm success\n", - "print(\"Loaded \" + str(len(candles)) + f\" rows of data for {pair} from {config['datadir']}\")\n", + "print(\"Loaded \" + str(len(candles)) + f\" rows of data for {pair} from {data_location}\")\n", "candles.head()" ] }, From 47874a452787360b610ab0f3d038de6e53cc5d53 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Wed, 12 Feb 2020 21:45:55 +0100 Subject: [PATCH 222/317] added logic to differentiate sell orders with double asterisk --- docs/telegram-usage.md | 2 +- freqtrade/rpc/rpc.py | 3 ++- freqtrade/rpc/telegram.py | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index c8ded4af5..f683ae8da 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -55,7 +55,7 @@ official commands. You can ask at any moment for help with `/help`. | `/reload_conf` | | Reloads the configuration file | `/show_config` | | Shows part of the current configuration with relevant settings to operation | `/status` | | Lists all open trades -| `/status table` | | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) +| `/status table` | | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**) | `/count` | | Displays number of trades used and available | `/profit` | | Display a summary of your profit/loss from close trades and some stats about your performance | `/forcesell ` | | Instantly sells the given trade (Ignoring `minimum_roi`). diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 07631f258..c182aad2b 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -169,7 +169,8 @@ class RPC: trades_list.append([ trade.id, trade.pair + '*' if (trade.open_order_id is not None - and trade.close_rate_requested is None) else '', + and trade.close_rate_requested is None) else '' + + '**' if (trade.close_rate_requested is not None) else '', shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), profit_str ]) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e3d4f54e7..d4ed5b189 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -561,6 +561,8 @@ class Telegram(RPC): "*/stop:* `Stops the trader`\n" \ "*/status [table]:* `Lists all open trades`\n" \ " *table :* `will display trades in a table`\n" \ + " pending buy orders are marked with an asterisk (*)\n" + " pending sell orders are marked with a double asterisk (**)\n" \ "*/profit:* `Lists cumulative profit from all finished trades`\n" \ "*/forcesell |all:* `Instantly sells the given trade or all trades, " \ "regardless of profit`\n" \ From f6db784a859b9a70a320e2c2da12dc47afce678e Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Wed, 12 Feb 2020 21:50:33 +0100 Subject: [PATCH 223/317] removed default to refresh argument in get_buy_rate and get_sell_rate --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ffd951ee2..455396352 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -234,7 +234,7 @@ class FreqtradeBot: return trades_created - def get_buy_rate(self, pair: str, refresh: bool = True, tick: Dict = None) -> float: + def get_buy_rate(self, pair: str, refresh: bool, tick: Dict = None) -> float: """ Calculates bid target between current ask price and last price :return: float: Price @@ -615,7 +615,7 @@ class FreqtradeBot: return trades_closed - def get_sell_rate(self, pair: str, refresh: bool = True) -> float: + def get_sell_rate(self, pair: str, refresh: bool) -> float: """ Get sell rate - either using get-ticker bid or first bid based on orderbook The orderbook portion is only used for rpc messaging, which would otherwise fail From 2e3b8cdba758a33abef2ce5f349335ff8194aa8a Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Wed, 12 Feb 2020 21:51:58 +0100 Subject: [PATCH 224/317] fixed flake8 issues on /help output --- freqtrade/rpc/telegram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index d4ed5b189..5603ab03c 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -561,8 +561,8 @@ class Telegram(RPC): "*/stop:* `Stops the trader`\n" \ "*/status [table]:* `Lists all open trades`\n" \ " *table :* `will display trades in a table`\n" \ - " pending buy orders are marked with an asterisk (*)\n" - " pending sell orders are marked with a double asterisk (**)\n" \ + " pending buy orders are marked with an asterisk (*)\n" \ + " pending sell orders are marked with a double asterisk (**)\n" \ "*/profit:* `Lists cumulative profit from all finished trades`\n" \ "*/forcesell |all:* `Instantly sells the given trade or all trades, " \ "regardless of profit`\n" \ From f09af888b153794ff135de9acedf493666124e9c Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Wed, 12 Feb 2020 21:55:38 +0100 Subject: [PATCH 225/317] modified get_buy/sell_rate refresh to true on notify_sell_cancel and notify_buy_cancel --- freqtrade/freqtradebot.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 455396352..158b631c1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -566,7 +566,7 @@ class FreqtradeBot: """ Sends rpc notification when a buy cancel occured. """ - current_rate = self.get_buy_rate(trade.pair, False) + current_rate = self.get_buy_rate(trade.pair, True) msg = { 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, @@ -1062,8 +1062,7 @@ class FreqtradeBot: """ profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) - # Use cached ticker here - it was updated seconds ago. - current_rate = self.get_sell_rate(trade.pair, False) + current_rate = self.get_sell_rate(trade.pair, True) profit_percent = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_percent > 0 else "loss" From 007cc94474c5f4bc0970f61b4716b00d8a8b8c92 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Wed, 12 Feb 2020 22:03:56 +0100 Subject: [PATCH 226/317] fixed tests to send refresh, since its no longer defaulted --- tests/test_freqtradebot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index c0af1f015..5ed4d296c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -921,7 +921,7 @@ def test_get_buy_rate(mocker, default_conf, ask, last, last_ab, expected) -> Non mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={'ask': ask, 'last': last})) - assert freqtrade.get_buy_rate('ETH/BTC') == expected + assert freqtrade.get_buy_rate('ETH/BTC', True) == expected def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: @@ -3524,7 +3524,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_buy_rate('ETH/BTC') == 0.043935 + assert freqtrade.get_buy_rate('ETH/BTC', True) == 0.043935 assert ticker_mock.call_count == 0 @@ -3549,7 +3549,7 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: freqtrade = FreqtradeBot(default_conf) # orderbook shall be used even if tickers would be lower. - assert freqtrade.get_buy_rate('ETH/BTC') != 0.042 + assert freqtrade.get_buy_rate('ETH/BTC', True) != 0.042 assert ticker_mock.call_count == 0 From 634bf2b15cf5078a1dba2fbd1ce0915346b30be5 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Thu, 13 Feb 2020 01:44:46 +0300 Subject: [PATCH 227/317] Docs: Fix checking of runmode --- docs/strategy-customization.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 688647c2b..07833da34 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -346,7 +346,7 @@ if self.dp: ``` python if self.dp: - if self.dp.runmode in ('live', 'dry_run'): + if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] @@ -422,7 +422,7 @@ from freqtrade.persistence import Trade The following example queries for the current pair and trades from today, however other filters can easily be added. ``` python -if self.config['runmode'] in ('live', 'dry_run'): +if self.config['runmode'].value in ('live', 'dry_run'): trades = Trade.get_trades([Trade.pair == metadata['pair'], Trade.open_date > datetime.utcnow() - timedelta(days=1), Trade.is_open == False, @@ -434,7 +434,7 @@ if self.config['runmode'] in ('live', 'dry_run'): Get amount of stake_currency currently invested in Trades: ``` python -if self.config['runmode'] in ('live', 'dry_run'): +if self.config['runmode'].value in ('live', 'dry_run'): total_stakes = Trade.total_open_trades_stakes() ``` @@ -442,7 +442,7 @@ Retrieve performance per pair. Returns a List of dicts per pair. ``` python -if self.config['runmode'] in ('live', 'dry_run'): +if self.config['runmode'].value in ('live', 'dry_run'): performance = Trade.get_overall_performance() ``` @@ -487,7 +487,7 @@ from datetime import timedelta, datetime, timezone # -------- # Within populate indicators (or populate_buy): -if self.config['runmode'] in ('live', 'dry_run'): +if self.config['runmode'].value in ('live', 'dry_run'): # fetch closed trades for the last 2 days trades = Trade.get_trades([Trade.pair == metadata['pair'], Trade.open_date > datetime.utcnow() - timedelta(days=2), From 81f849811fdcf4a2273f56ba2d7851aec66990a6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Feb 2020 06:30:59 +0100 Subject: [PATCH 228/317] Initcap Freqtrade --- docs/strategy_analysis_example.md | 2 +- freqtrade/templates/strategy_analysis_example.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 93e84122b..f3f9b46c3 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -1,6 +1,6 @@ # Strategy analysis example -Debugging a strategy can be time-consuming. FreqTrade offers helper functions to visualize raw data. +Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data. ## Setup diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 91e132380..021056df2 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -6,7 +6,7 @@ "source": [ "# Strategy analysis example\n", "\n", - "Debugging a strategy can be time-consuming. FreqTrade offers helper functions to visualize raw data." + "Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data." ] }, { From 86592c3ba1c0f4b626d7d4edb471a320d120dd13 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Feb 2020 06:51:52 +0100 Subject: [PATCH 229/317] Fix /help from telegram --- freqtrade/rpc/telegram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 5603ab03c..abd322293 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -561,8 +561,8 @@ class Telegram(RPC): "*/stop:* `Stops the trader`\n" \ "*/status [table]:* `Lists all open trades`\n" \ " *table :* `will display trades in a table`\n" \ - " pending buy orders are marked with an asterisk (*)\n" \ - " pending sell orders are marked with a double asterisk (**)\n" \ + " `pending buy orders are marked with an asterisk (*)`\n" \ + " `pending sell orders are marked with a double asterisk (**)`\n" \ "*/profit:* `Lists cumulative profit from all finished trades`\n" \ "*/forcesell |all:* `Instantly sells the given trade or all trades, " \ "regardless of profit`\n" \ From ccc923975138e47440e5ce48fd6fb38c1f0691c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Feb 2020 07:02:12 +0100 Subject: [PATCH 230/317] Reduce indentation of help --- freqtrade/rpc/telegram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index abd322293..e3958b31a 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -561,8 +561,8 @@ class Telegram(RPC): "*/stop:* `Stops the trader`\n" \ "*/status [table]:* `Lists all open trades`\n" \ " *table :* `will display trades in a table`\n" \ - " `pending buy orders are marked with an asterisk (*)`\n" \ - " `pending sell orders are marked with a double asterisk (**)`\n" \ + " `pending buy orders are marked with an asterisk (*)`\n" \ + " `pending sell orders are marked with a double asterisk (**)`\n" \ "*/profit:* `Lists cumulative profit from all finished trades`\n" \ "*/forcesell |all:* `Instantly sells the given trade or all trades, " \ "regardless of profit`\n" \ From a93bc74eff01cebaed7e1e9838a53b526178913d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Feb 2020 07:04:37 +0100 Subject: [PATCH 231/317] Update documentation ... --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index c0404d647..98300c5fa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -40,7 +40,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | Parameter | Description | |------------|-------------| -| `max_open_trades` | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades). [More information below](#configuring-amount-per-trade).
***Datatype:*** *Positive integer or -1.* +| `max_open_trades` | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades). [More information below](#configuring-amount-per-trade).
**Datatype:** Positive integer or -1. | `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *String* | `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Positive float or `"unlimited"`.* | `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade).
*Defaults to `0.99` 99%).*
***Datatype:*** *Positive float between `0.1` and `1.0`.* From 02148a1df203d8512dc6987ce7ac4ad5111211a2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Feb 2020 15:09:09 +0100 Subject: [PATCH 232/317] Fix datatype styling issues --- docs/configuration.md | 132 +++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 98300c5fa..fd686834f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -41,74 +41,74 @@ Mandatory parameters are marked as **Required**, which means that they are requi | Parameter | Description | |------------|-------------| | `max_open_trades` | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades). [More information below](#configuring-amount-per-trade).
**Datatype:** Positive integer or -1. -| `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *String* -| `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Positive float or `"unlimited"`.* -| `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade).
*Defaults to `0.99` 99%).*
***Datatype:*** *Positive float between `0.1` and `1.0`.* -| `amend_last_stake_amount` | Use reduced last stake amount if necessary. [More information below](#configuring-amount-per-trade).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `last_stake_amount_min_ratio` | Defines minimum stake amount that has to be left and executed. Applies only to the last stake amount when it's amended to a reduced value (i.e. if `amend_last_stake_amount` is set to `true`). [More information below](#configuring-amount-per-trade).
*Defaults to `0.5`.*
***Datatype:*** *Float (as ratio)* -| `amount_reserve_percent` | Reserve some amount in min pair stake amount. The bot will reserve `amount_reserve_percent` + stoploss value when calculating min pair stake amount in order to avoid possible trade refusals.
*Defaults to `0.05` (5%).*
***Datatype:*** *Positive Float as ratio.* -| `ticker_interval` | The ticker interval to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *String* -| `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
***Datatype:*** *String* -| `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
***Datatype:*** *Boolean* -| `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in the Dry Run mode.
*Defaults to `1000`.*
***Datatype:*** *Float* -| `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `minimal_roi` | **Required.** Set the threshold in percent the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* -| `stoploss` | **Required.** Value of the stoploss in percent used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Float (as ratio)* -| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Boolean* -| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Float* -| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
***Datatype:*** *Float* -| `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Integer* -| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Integer* +| `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String +| `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Positive float or `"unlimited"`. +| `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade).
*Defaults to `0.99` 99%).*
**Datatype:** Positive float between `0.1` and `1.0`. +| `amend_last_stake_amount` | Use reduced last stake amount if necessary. [More information below](#configuring-amount-per-trade).
*Defaults to `false`.*
**Datatype:** Boolean +| `last_stake_amount_min_ratio` | Defines minimum stake amount that has to be left and executed. Applies only to the last stake amount when it's amended to a reduced value (i.e. if `amend_last_stake_amount` is set to `true`). [More information below](#configuring-amount-per-trade).
*Defaults to `0.5`.*
**Datatype:** Float (as ratio) +| `amount_reserve_percent` | Reserve some amount in min pair stake amount. The bot will reserve `amount_reserve_percent` + stoploss value when calculating min pair stake amount in order to avoid possible trade refusals.
*Defaults to `0.05` (5%).*
**Datatype:** Positive Float as ratio. +| `ticker_interval` | The ticker interval to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String +| `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
**Datatype:** String +| `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
**Datatype:** Boolean +| `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in the Dry Run mode.
*Defaults to `1000`.*
**Datatype:** Float +| `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `minimal_roi` | **Required.** Set the threshold in percent the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict +| `stoploss` | **Required.** Value of the stoploss in percent used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Float (as ratio) +| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Boolean +| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Float +| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
**Datatype:** Float +| `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer +| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer | `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#buy-price-without-orderbook). -| `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled).
***Datatype:*** *Boolean* -| `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids to buy. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in [Order Book Bids](#buy-price-with-orderbook-enabled).
*Defaults to `1`.*
***Datatype:*** *Positive Integer* -| `bid_strategy. check_depth_of_market.enabled` | Do not buy if the difference of buy orders and sell orders is met in Order Book. [Check market depth](#check-depth-of-market).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `bid_strategy. check_depth_of_market.bids_to_ask_delta` | The difference ratio of buy orders and sell orders found in Order Book. A value below 1 means sell order size is greater, while value greater than 1 means buy order size is higher. [Check market depth](#check-depth-of-market)
*Defaults to `0`.*
***Datatype:*** *Float (as ratio)* -| `ask_strategy.use_order_book` | Enable selling of open trades using [Order Book Asks](#sell-price-with-orderbook-enabled).
***Datatype:*** *Boolean* -| `ask_strategy.order_book_min` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
***Datatype:*** *Positive Integer* -| `ask_strategy.order_book_max` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
***Datatype:*** *Positive Integer* -| `ask_strategy.use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
***Datatype:*** *Boolean* -| `ask_strategy.sell_profit_only` | Wait until the bot makes a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `ask_strategy.ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* -| `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* -| `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
***Datatype:*** *String* -| `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.
***Datatype:*** *Boolean* -| `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#dynamic-pairlists)).
***Datatype:*** *List* -| `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#dynamic-pairlists)).
***Datatype:*** *List* -| `exchange.ccxt_config` | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
***Datatype:*** *Dict* -| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
***Datatype:*** *Dict* -| `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
***Datatype:*** *Positive Integer* +| `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled).
**Datatype:** Boolean +| `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids to buy. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in [Order Book Bids](#buy-price-with-orderbook-enabled).
*Defaults to `1`.*
**Datatype:** Positive Integer +| `bid_strategy. check_depth_of_market.enabled` | Do not buy if the difference of buy orders and sell orders is met in Order Book. [Check market depth](#check-depth-of-market).
*Defaults to `false`.*
**Datatype:** Boolean +| `bid_strategy. check_depth_of_market.bids_to_ask_delta` | The difference ratio of buy orders and sell orders found in Order Book. A value below 1 means sell order size is greater, while value greater than 1 means buy order size is higher. [Check market depth](#check-depth-of-market)
*Defaults to `0`.*
**Datatype:** Float (as ratio) +| `ask_strategy.use_order_book` | Enable selling of open trades using [Order Book Asks](#sell-price-with-orderbook-enabled).
**Datatype:** Boolean +| `ask_strategy.order_book_min` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
**Datatype:** Positive Integer +| `ask_strategy.order_book_max` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
**Datatype:** Positive Integer +| `ask_strategy.use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean +| `ask_strategy.sell_profit_only` | Wait until the bot makes a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `ask_strategy.ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict +| `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict +| `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
**Datatype:** String +| `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.
**Datatype:** Boolean +| `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#dynamic-pairlists)).
**Datatype:** List +| `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#dynamic-pairlists)).
**Datatype:** List +| `exchange.ccxt_config` | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict +| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict +| `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
**Datatype:** Positive Integer | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. -| `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
***Datatype:*** *Boolean* -| `pairlists` | Define one or more pairlists to be used. [More information below](#dynamic-pairlists).
*Defaults to `StaticPairList`.*
***Datatype:*** *List of Dicts* -| `telegram.enabled` | Enable the usage of Telegram.
***Datatype:*** *Boolean* -| `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `webhook.enabled` | Enable usage of Webhook notifications
***Datatype:*** *Boolean* -| `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* -| `webhook.webhookbuy` | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* -| `webhook.webhooksell` | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* -| `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* -| `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Boolean* -| `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *IPv4* -| `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Integer between 1024 and 65535* -| `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
***Datatype:*** *String, SQLAlchemy connect string* -| `initial_state` | Defines the initial application state. More information below.
*Defaults to `stopped`.*
***Datatype:*** *Enum, either `stopped` or `running`* -| `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
***Datatype:*** *Boolean* -| `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
***Datatype:*** *ClassName* -| `strategy_path` | Adds an additional strategy lookup path (must be a directory).
***Datatype:*** *String* -| `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
***Datatype:*** *Positive Integer* -| `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages.
*Defaults to `60` seconds.*
***Datatype:*** *Positive Integer or 0* -| `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
***Datatype:*** *Boolean* -| `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
***Datatype:*** *String* -| `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
***Datatype:*** *String* +| `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean +| `pairlists` | Define one or more pairlists to be used. [More information below](#dynamic-pairlists).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts +| `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean +| `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `webhook.enabled` | Enable usage of Webhook notifications
**Datatype:** Boolean +| `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| `webhook.webhookbuy` | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| `webhook.webhooksell` | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** Boolean +| `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** IPv4 +| `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** Integer between 1024 and 65535 +| `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
**Datatype:** String, SQLAlchemy connect string +| `initial_state` | Defines the initial application state. More information below.
*Defaults to `stopped`.*
**Datatype:** Enum, either `stopped` or `running` +| `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
**Datatype:** Boolean +| `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
**Datatype:** ClassName +| `strategy_path` | Adds an additional strategy lookup path (must be a directory).
**Datatype:** String +| `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
**Datatype:** Positive Intege +| `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages.
*Defaults to `60` seconds.*
**Datatype:** Positive Integer or 0 +| `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
**Datatype:** Boolean +| `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
**Datatype:** String +| `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
**Datatype:** String ### Parameters in the strategy From a0a14a107820df972d1e02b42267093f58748f1e Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 01:08:17 +0300 Subject: [PATCH 233/317] freqtrade/templates/subtemplates/exchange_bittrex.j2 --- freqtrade/templates/subtemplates/exchange_bittrex.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/templates/subtemplates/exchange_bittrex.j2 b/freqtrade/templates/subtemplates/exchange_bittrex.j2 index 7a7e8e291..7b27318ca 100644 --- a/freqtrade/templates/subtemplates/exchange_bittrex.j2 +++ b/freqtrade/templates/subtemplates/exchange_bittrex.j2 @@ -1,3 +1,10 @@ +"order_types": { + "buy": "limit", + "sell": "limit", + "emergencysell": "limit", + "stoploss": "limit", + "stoploss_on_exchange": false +}, "exchange": { "name": "{{ exchange_name | lower }}", "key": "{{ exchange_key }}", From 749463e4b7e9f9b1c475725a54b71d181b3a7a76 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 03:05:07 +0300 Subject: [PATCH 234/317] Adjust message in main.py --- freqtrade/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index a75eeebed..08bdc5e32 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -38,8 +38,8 @@ def main(sysargv: List[str] = None) -> None: # No subcommand was issued. raise OperationalException( "Usage of Freqtrade requires a subcommand to be specified.\n" - "To have the previous behavior (bot executing trades in live/dry-run modes, " - "depending on the value of the `dry_run` setting in the config), run freqtrade " + "To have the bot executing trades in live/dry-run modes, " + "depending on the value of the `dry_run` setting in the config, run Freqtrade " "as `freqtrade trade [options...]`.\n" "To see the full list of options available, please use " "`freqtrade --help` or `freqtrade --help`." From 36ef5c6bdff2562e158bbd659db7bc6c67c00560 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 04:05:17 +0300 Subject: [PATCH 235/317] Get rid of delete_trades method in Freqtradebot --- freqtrade/freqtradebot.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 158b631c1..bf2697efa 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -860,11 +860,6 @@ class FreqtradeBot: order_type = self.strategy.order_types['sell'] self._notify_sell_cancel(trade, order_type) - def delete_trade(self, trade: Trade) -> None: - """Delete trade in database""" - Trade.session.delete(trade) - Trade.session.flush() - def handle_timedout_limit_buy(self, trade: Trade, order: Dict) -> bool: """ Buy timeout - cancel order @@ -882,7 +877,8 @@ class FreqtradeBot: if corder.get('remaining', order['remaining']) == order['amount']: # if trade is not partially completed, just delete the trade - self.delete_trade(trade) + Trade.session.delete(trade) + Trade.session.flush() return True # if trade is partially complete, edit the stake details for the trade From 20c21b42d50ce08fe4157c151c340a6be771b7f2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 06:23:03 +0300 Subject: [PATCH 236/317] Move rpc send to be after db session add/flash --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 158b631c1..1284b2e9b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -527,8 +527,6 @@ class FreqtradeBot: ticker_interval=timeframe_to_minutes(self.config['ticker_interval']) ) - self._notify_buy(trade, order_type) - # Update fees if order is closed if order_status == 'closed': self.update_trade_state(trade, order) @@ -539,6 +537,8 @@ class FreqtradeBot: # Updating wallets self.wallets.update() + self._notify_buy(trade, order_type) + return True def _notify_buy(self, trade: Trade, order_type: str) -> None: From 9cbf8c5f008520b99016a7a63fe299b4b0dcb821 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 21:15:36 +0300 Subject: [PATCH 237/317] Add status for listed strategies --- freqtrade/commands/list_commands.py | 11 +++++++++-- freqtrade/resolvers/iresolver.py | 20 +++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index f2b6bf995..b6ff682e6 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -43,10 +43,17 @@ def start_list_strategies(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) - strategies = StrategyResolver.search_all_objects(directory) + strategies = StrategyResolver.search_all_objects(directory, not args['print_one_column']) # Sort alphabetically strategies = sorted(strategies, key=lambda x: x['name']) - strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies] + names = [s['name'] for s in strategies] + strats_to_print = [{ + 'name': s['name'] if s['name'] else "--", + 'location': s['location'].name, + 'status': ("LOAD FAILED" if s['class'] is None + else "OK" if names.count(s['name']) == 1 + else "DUPLICATED NAME") + } for s in strategies] if args['print_one_column']: print('\n'.join([s['name'] for s in strategies])) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index a75c45933..8b5aa1dff 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -41,11 +41,15 @@ class IResolver: @classmethod def _get_valid_object(cls, module_path: Path, - object_name: Optional[str]) -> Generator[Any, None, None]: + object_name: Optional[str], + enum_failed: bool = False) -> Union[Generator[Any, None, None], + Tuple[None]]: """ Generator returning objects with matching object_type and object_name in the path given. :param module_path: absolute path to the module :param object_name: Class name of the object + :param enum_failed: If True, will return None for modules which fail. + Otherwise, failing modules are skipped. :return: generator containing matching objects """ @@ -58,6 +62,8 @@ class IResolver: except (ModuleNotFoundError, SyntaxError) as err: # Catch errors in case a specific module is not installed logger.warning(f"Could not import {module_path} due to '{err}'") + if enum_failed: + return (None, ) valid_objects_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) @@ -136,10 +142,13 @@ class IResolver: ) @classmethod - def search_all_objects(cls, directory: Path) -> List[Dict[str, Any]]: + def search_all_objects(cls, directory: Path, + enum_failed: bool) -> List[Dict[str, Any]]: """ Searches a directory for valid objects :param directory: Path to search + :param enum_failed: If True, will return None for modules which fail. + Otherwise, failing modules are skipped. :return: List of dicts containing 'name', 'class' and 'location' entires """ logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'") @@ -151,10 +160,11 @@ class IResolver: continue module_path = entry.resolve() logger.debug(f"Path {module_path}") - for obj in cls._get_valid_object(module_path, object_name=None): + for obj in cls._get_valid_object(module_path, object_name=None, + enum_failed=enum_failed): objects.append( - {'name': obj.__name__, - 'class': obj, + {'name': obj.__name__ if obj is not None else '', + 'class': obj if obj is not None else None, 'location': entry, }) return objects From a2d7f8a70dc8e907c83ab4e958783708174ab07f Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 21:24:30 +0300 Subject: [PATCH 238/317] Split tabular printing into sep. helper function --- freqtrade/commands/list_commands.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index b6ff682e6..782cd074e 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -3,7 +3,7 @@ import logging import sys from collections import OrderedDict from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, List import rapidjson from tabulate import tabulate @@ -36,6 +36,19 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") +def _print_objs_tabular(objs: List) -> None: + names = [s['name'] for s in objs] + strats_to_print = [{ + 'name': s['name'] if s['name'] else "--", + 'location': s['location'].name, + 'status': ("LOAD FAILED" if s['class'] is None + else "OK" if names.count(s['name']) == 1 + else "DUPLICATED NAME") + } for s in objs] + + print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) + + def start_list_strategies(args: Dict[str, Any]) -> None: """ Print files with Strategy custom classes available in the directory @@ -46,19 +59,11 @@ def start_list_strategies(args: Dict[str, Any]) -> None: strategies = StrategyResolver.search_all_objects(directory, not args['print_one_column']) # Sort alphabetically strategies = sorted(strategies, key=lambda x: x['name']) - names = [s['name'] for s in strategies] - strats_to_print = [{ - 'name': s['name'] if s['name'] else "--", - 'location': s['location'].name, - 'status': ("LOAD FAILED" if s['class'] is None - else "OK" if names.count(s['name']) == 1 - else "DUPLICATED NAME") - } for s in strategies] if args['print_one_column']: print('\n'.join([s['name'] for s in strategies])) else: - print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) + _print_objs_tabular(strategies) def start_list_hyperopts(args: Dict[str, Any]) -> None: From 9dafc2f3c84819cf95bda82ddb0cc8ec380ede3d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Feb 2020 20:38:29 +0100 Subject: [PATCH 239/317] Load config.json from user_data first --- freqtrade/commands/arguments.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index fe6f49039..d37870ea0 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -6,8 +6,8 @@ from functools import partial from pathlib import Path from typing import Any, Dict, List, Optional -from freqtrade import constants from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS +from freqtrade.constants import DEFAULT_CONFIG ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"] @@ -107,10 +107,19 @@ class Arguments: # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) # Allow no-config for certain commands (like downloading / plotting) - if ('config' in parsed_arg and parsed_arg.config is None and - ((Path.cwd() / constants.DEFAULT_CONFIG).is_file() or - not ('command' in parsed_arg and parsed_arg.command in NO_CONF_REQURIED))): - parsed_arg.config = [constants.DEFAULT_CONFIG] + if ('config' in parsed_arg and parsed_arg.config is None): + conf_required = ('command' in parsed_arg and parsed_arg.command in NO_CONF_REQURIED) + + if 'user_data_dir' in parsed_arg and parsed_arg.user_data_dir is not None: + # Try loading from "user_data/config.json" + cfgfile = Path(parsed_arg.user_data_dir) / DEFAULT_CONFIG + if cfgfile.is_file() or not conf_required: + parsed_arg.config = [str(cfgfile)] + else: + # Else use "config.json". + cfgfile = Path.cwd() / DEFAULT_CONFIG + if cfgfile.is_file() or not conf_required: + parsed_arg.config = [DEFAULT_CONFIG] return parsed_arg From be4a9b5f4b0f9c4026ed043876bbcfb516a83a0d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 14 Feb 2020 19:37:20 +0100 Subject: [PATCH 240/317] Lowercase freqtrade --- CONTRIBUTING.md | 4 ++-- docs/configuration.md | 2 +- docs/developer.md | 4 ++-- docs/hyperopt.md | 2 +- docs/rest-api.md | 2 +- freqtrade/commands/cli_options.py | 3 ++- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c83437f6..90594866a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,11 +109,11 @@ Exceptions: Contributors may be given commit privileges. Preference will be given to those with: -1. Past contributions to FreqTrade and other related open-source projects. Contributions to FreqTrade include both code (both accepted and pending) and friendly participation in the issue tracker and Pull request reviews. Quantity and quality are considered. +1. Past contributions to Freqtrade and other related open-source projects. Contributions to Freqtrade include both code (both accepted and pending) and friendly participation in the issue tracker and Pull request reviews. Quantity and quality are considered. 1. A coding style that the other core committers find simple, minimal, and clean. 1. Access to resources for cross-platform development and testing. 1. Time to devote to the project regularly. -Being a Committer does not grant write permission on `develop` or `master` for security reasons (Users trust FreqTrade with their Exchange API keys). +Being a Committer does not grant write permission on `develop` or `master` for security reasons (Users trust Freqtrade with their Exchange API keys). After being Committer for some time, a Committer may be named Core Committer and given full repository access. diff --git a/docs/configuration.md b/docs/configuration.md index 211f7a04c..0f0279eb9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -672,7 +672,7 @@ freqtrade ## Embedding Strategies -FreqTrade provides you with with an easy way to embed the strategy into your configuration file. +Freqtrade provides you with with an easy way to embed the strategy into your configuration file. This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. diff --git a/docs/developer.md b/docs/developer.md index c679b8a49..b128ffd2b 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -1,6 +1,6 @@ # Development Help -This page is intended for developers of FreqTrade, people who want to contribute to the FreqTrade codebase or documentation, or people who want to understand the source code of the application they're running. +This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running. All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel in [slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) where you can ask questions. @@ -153,7 +153,7 @@ In VolumePairList, this implements different methods of sorting, does early vali ## Implement a new Exchange (WIP) !!! Note - This section is a Work in Progress and is not a complete guide on how to test a new exchange with FreqTrade. + This section is a Work in Progress and is not a complete guide on how to test a new exchange with Freqtrade. Most exchanges supported by CCXT should work out of the box. diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 3e10f66da..401811a1b 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -182,7 +182,7 @@ add it to the `populate_indicators()` method in your custom hyperopt file. Each hyperparameter tuning requires a target. This is usually defined as a loss function (sometimes also called objective function), which should decrease for more desirable results, and increase for bad results. -By default, FreqTrade uses a loss function, which has been with freqtrade since the beginning and optimizes mostly for short trade duration and avoiding losses. +By default, Freqtrade uses a loss function, which has been with freqtrade since the beginning and optimizes mostly for short trade duration and avoiding losses. A different loss function can be specified by using the `--hyperopt-loss ` argument. This class should be in its own file within the `user_data/hyperopts/` directory. diff --git a/docs/rest-api.md b/docs/rest-api.md index 187a71c97..b68364f39 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -74,7 +74,7 @@ docker run -d \ ## Consuming the API You can consume the API by using the script `scripts/rest_client.py`. -The client script only requires the `requests` module, so FreqTrade does not need to be installed on the system. +The client script only requires the `requests` module, so Freqtrade does not need to be installed on the system. ``` bash python3 scripts/rest_client.py [optional parameters] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 1776955b1..cdc8cb8f1 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -59,7 +59,8 @@ AVAILABLE_CLI_OPTIONS = { ), "config": Arg( '-c', '--config', - help=f'Specify configuration file (default: `{constants.DEFAULT_CONFIG}`). ' + help=f'Specify configuration file (default: `userdir/{constants.DEFAULT_CONFIG}` ' + f'or `config.json` whichever exists). ' f'Multiple --config options may be used. ' f'Can be set to `-` to read config from stdin.', action='append', From 1bc26fd07a11bf064098e544e4a42063d3294b90 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 21:46:22 +0300 Subject: [PATCH 241/317] Add printing statuses for list-hyperopts --- freqtrade/commands/list_commands.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 782cd074e..a2ac388b0 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -38,7 +38,7 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: def _print_objs_tabular(objs: List) -> None: names = [s['name'] for s in objs] - strats_to_print = [{ + objss_to_print = [{ 'name': s['name'] if s['name'] else "--", 'location': s['location'].name, 'status': ("LOAD FAILED" if s['class'] is None @@ -46,7 +46,7 @@ def _print_objs_tabular(objs: List) -> None: else "DUPLICATED NAME") } for s in objs] - print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) + print(tabulate(objss_to_print, headers='keys', tablefmt='pipe')) def start_list_strategies(args: Dict[str, Any]) -> None: @@ -56,14 +56,14 @@ def start_list_strategies(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) - strategies = StrategyResolver.search_all_objects(directory, not args['print_one_column']) + strategy_objs = StrategyResolver.search_all_objects(directory, not args['print_one_column']) # Sort alphabetically - strategies = sorted(strategies, key=lambda x: x['name']) + strategy_objs = sorted(strategy_objs, key=lambda x: x['name']) if args['print_one_column']: - print('\n'.join([s['name'] for s in strategies])) + print('\n'.join([s['name'] for s in strategy_objs])) else: - _print_objs_tabular(strategies) + _print_objs_tabular(strategy_objs) def start_list_hyperopts(args: Dict[str, Any]) -> None: @@ -75,15 +75,14 @@ def start_list_hyperopts(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) directory = Path(config.get('hyperopt_path', config['user_data_dir'] / USERPATH_HYPEROPTS)) - hyperopts = HyperOptResolver.search_all_objects(directory) + hyperopt_objs = HyperOptResolver.search_all_objects(directory, not args['print_one_column']) # Sort alphabetically - hyperopts = sorted(hyperopts, key=lambda x: x['name']) - hyperopts_to_print = [{'name': s['name'], 'location': s['location'].name} for s in hyperopts] + hyperopt_objs = sorted(hyperopt_objs, key=lambda x: x['name']) if args['print_one_column']: - print('\n'.join([s['name'] for s in hyperopts])) + print('\n'.join([s['name'] for s in hyperopt_objs])) else: - print(tabulate(hyperopts_to_print, headers='keys', tablefmt='pipe')) + _print_objs_tabular(hyperopt_objs) def start_list_timeframes(args: Dict[str, Any]) -> None: From c92e1d97d65b796a061d19081f701cac6aa59ce3 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 21:52:02 +0300 Subject: [PATCH 242/317] Attempt to make mypy happy --- freqtrade/resolvers/iresolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 8b5aa1dff..d674daa9a 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -89,7 +89,7 @@ class IResolver: continue module_path = entry.resolve() - obj = next(cls._get_valid_object(module_path, object_name), None) + obj = next(cls._get_valid_object(module_path, object_name), None) # noqa if obj: return (obj, module_path) From 5efbdd25a7776c55eb8fbfb43a0e4b09213b4c41 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 14 Feb 2020 20:04:05 +0100 Subject: [PATCH 243/317] Properly default to user_data/config.json if it exists --- docs/bot-usage.md | 52 +++++++++++++++++++++------------ freqtrade/commands/arguments.py | 10 +++++-- tests/test_arguments.py | 24 +++++++++++++-- tests/test_main.py | 5 +++- 4 files changed, 66 insertions(+), 25 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 56e6008a1..dbc111d44 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -58,9 +58,10 @@ Common arguments: details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -71,6 +72,7 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. +. ``` @@ -242,12 +244,15 @@ optional arguments: Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -280,7 +285,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--hyperopt NAME] [--hyperopt-path PATH] [--eps] [-e INT] - [--spaces {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]] + [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]] [--dmmp] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] [--continue] [--hyperopt-loss NAME] @@ -308,9 +313,9 @@ optional arguments: Allow buying the same pair multiple times (position stacking). -e INT, --epochs INT Specify number of epochs (default: 100). - --spaces {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...] + --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...] Specify which parameters to hyperopt. Space-separated - list. Default: `all`. + list. --dmmp, --disable-max-market-positions Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high @@ -338,16 +343,20 @@ optional arguments: target for optimization is different. Built-in Hyperopt-loss-functions are: DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, - SharpeHyperOptLossDaily (default: `DefaultHyperOptLoss`). + SharpeHyperOptLossDaily.(default: + `DefaultHyperOptLoss`). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -358,6 +367,7 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. + ``` ## Edge commands @@ -394,12 +404,15 @@ optional arguments: Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -410,6 +423,7 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. + ``` To understand edge and how to read the results, please read the [edge documentation](edge.md). diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index d37870ea0..580c9e298 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -111,10 +111,14 @@ class Arguments: conf_required = ('command' in parsed_arg and parsed_arg.command in NO_CONF_REQURIED) if 'user_data_dir' in parsed_arg and parsed_arg.user_data_dir is not None: + user_dir = parsed_arg.user_data_dir + else: + # Default case + user_dir = 'user_data' # Try loading from "user_data/config.json" - cfgfile = Path(parsed_arg.user_data_dir) / DEFAULT_CONFIG - if cfgfile.is_file() or not conf_required: - parsed_arg.config = [str(cfgfile)] + cfgfile = Path(user_dir) / DEFAULT_CONFIG + if cfgfile.is_file(): + parsed_arg.config = [str(cfgfile)] else: # Else use "config.json". cfgfile = Path.cwd() / DEFAULT_CONFIG diff --git a/tests/test_arguments.py b/tests/test_arguments.py index 60da0082a..61bca04a4 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -18,7 +18,8 @@ def test_parse_args_none() -> None: assert isinstance(arguments.parser, argparse.ArgumentParser) -def test_parse_args_defaults() -> None: +def test_parse_args_defaults(mocker) -> None: + mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True])) args = Arguments(['trade']).get_parsed_arg() assert args["config"] == ['config.json'] assert args["strategy_path"] is None @@ -26,6 +27,25 @@ def test_parse_args_defaults() -> None: assert args["verbosity"] == 0 +def test_parse_args_default_userdatadir(mocker) -> None: + mocker.patch.object(Path, "is_file", MagicMock(return_value=True)) + args = Arguments(['trade']).get_parsed_arg() + # configuration defaults to user_data if that is available. + assert args["config"] == ['user_data/config.json'] + assert args["strategy_path"] is None + assert args["datadir"] is None + assert args["verbosity"] == 0 + + +def test_parse_args_userdatadir(mocker) -> None: + mocker.patch.object(Path, "is_file", MagicMock(return_value=True)) + args = Arguments(['trade', '--user-data-dir', 'user_data']).get_parsed_arg() + # configuration defaults to user_data if that is available. + assert args["config"] == ['user_data/config.json'] + assert args["strategy_path"] is None + assert args["datadir"] is None + assert args["verbosity"] == 0 + def test_parse_args_config() -> None: args = Arguments(['trade', '-c', '/dev/null']).get_parsed_arg() assert args["config"] == ['/dev/null'] @@ -208,7 +228,7 @@ def test_config_notrequired(mocker) -> None: assert pargs["config"] is None # When file exists: - mocker.patch.object(Path, "is_file", MagicMock(return_value=True)) + mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True])) args = [ 'download-data', ] diff --git a/tests/test_main.py b/tests/test_main.py index 1229f748a..70b784002 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,8 +5,9 @@ from unittest.mock import MagicMock, PropertyMock import pytest +from pathlib import Path from freqtrade.commands import Arguments -from freqtrade.exceptions import OperationalException, FreqtradeException +from freqtrade.exceptions import FreqtradeException, OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.main import main from freqtrade.state import State @@ -26,6 +27,7 @@ def test_parse_args_backtesting(mocker) -> None: Test that main() can start backtesting and also ensure we can pass some specific arguments further argument parsing is done in test_arguments.py """ + mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True])) backtesting_mock = mocker.patch('freqtrade.commands.start_backtesting') backtesting_mock.__name__ = PropertyMock("start_backtesting") # it's sys.exit(0) at the end of backtesting @@ -42,6 +44,7 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: + mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True])) hyperopt_mock = mocker.patch('freqtrade.commands.start_hyperopt', MagicMock()) hyperopt_mock.__name__ = PropertyMock("start_hyperopt") # it's sys.exit(0) at the end of hyperopt From d5a298bbb7dd3db6be8f16c7ec1376fc7c677901 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 14 Feb 2020 20:12:26 +0100 Subject: [PATCH 244/317] Add sentence from suggestion --- docs/strategy_analysis_example.md | 1 + freqtrade/templates/strategy_analysis_example.ipynb | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index f3f9b46c3..53b35ca09 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -1,6 +1,7 @@ # Strategy analysis example Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data. +The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location. ## Setup diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 021056df2..399235cfe 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -6,7 +6,8 @@ "source": [ "# Strategy analysis example\n", "\n", - "Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data." + "Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data.\n", + "The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location." ] }, { From ecca7164d91a41ee252c93c2533e6e7737d0db71 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 14 Feb 2020 20:13:36 +0100 Subject: [PATCH 245/317] Fix small issue --- tests/test_arguments.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_arguments.py b/tests/test_arguments.py index 61bca04a4..22383661b 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -46,6 +46,7 @@ def test_parse_args_userdatadir(mocker) -> None: assert args["datadir"] is None assert args["verbosity"] == 0 + def test_parse_args_config() -> None: args = Arguments(['trade', '-c', '/dev/null']).get_parsed_arg() assert args["config"] == ['/dev/null'] From f024cc40d3808bfd75d55e75f8a8370b86cae57e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 14 Feb 2020 20:21:09 +0100 Subject: [PATCH 246/317] Fix windows test failure --- tests/test_arguments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_arguments.py b/tests/test_arguments.py index 22383661b..0052a61d0 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -31,7 +31,7 @@ def test_parse_args_default_userdatadir(mocker) -> None: mocker.patch.object(Path, "is_file", MagicMock(return_value=True)) args = Arguments(['trade']).get_parsed_arg() # configuration defaults to user_data if that is available. - assert args["config"] == ['user_data/config.json'] + assert args["config"] == [str(Path('user_data/config.json'))] assert args["strategy_path"] is None assert args["datadir"] is None assert args["verbosity"] == 0 @@ -41,7 +41,7 @@ def test_parse_args_userdatadir(mocker) -> None: mocker.patch.object(Path, "is_file", MagicMock(return_value=True)) args = Arguments(['trade', '--user-data-dir', 'user_data']).get_parsed_arg() # configuration defaults to user_data if that is available. - assert args["config"] == ['user_data/config.json'] + assert args["config"] == [str(Path('user_data/config.json'))] assert args["strategy_path"] is None assert args["datadir"] is None assert args["verbosity"] == 0 From e598c769d448db0a03c7c7df5d470d3a389d8cee Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 22:28:49 +0300 Subject: [PATCH 247/317] Add colorization --- freqtrade/commands/arguments.py | 4 ++-- freqtrade/commands/list_commands.py | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index fe6f49039..063a152fe 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -30,9 +30,9 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] -ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column"] +ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized"] -ARGS_LIST_HYPEROPTS = ["hyperopt_path", "print_one_column"] +ARGS_LIST_HYPEROPTS = ["hyperopt_path", "print_one_column", "print_colorized"] ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"] diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index a2ac388b0..6a2ccbfcf 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -5,6 +5,8 @@ from collections import OrderedDict from pathlib import Path from typing import Any, Dict, List +from colorama import init as colorama_init +from colorama import Fore, Style import rapidjson from tabulate import tabulate @@ -36,14 +38,23 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") -def _print_objs_tabular(objs: List) -> None: +def _print_objs_tabular(objs: List, print_colorized: bool) -> None: + if print_colorized: + colorama_init(autoreset=True) + names = [s['name'] for s in objs] objss_to_print = [{ 'name': s['name'] if s['name'] else "--", 'location': s['location'].name, - 'status': ("LOAD FAILED" if s['class'] is None - else "OK" if names.count(s['name']) == 1 - else "DUPLICATED NAME") + 'status': (((Fore.RED if print_colorized else '') + + "LOAD FAILED" + (Style.RESET_ALL if print_colorized else '')) + if s['class'] is None + else ((Fore.GREEN if print_colorized else '') + + "OK" + (Style.RESET_ALL if print_colorized else '')) + if names.count(s['name']) == 1 + else ((Fore.YELLOW if print_colorized else '') + + "DUPLICATED NAME" + + (Style.RESET_ALL if print_colorized else ''))) } for s in objs] print(tabulate(objss_to_print, headers='keys', tablefmt='pipe')) @@ -63,7 +74,7 @@ def start_list_strategies(args: Dict[str, Any]) -> None: if args['print_one_column']: print('\n'.join([s['name'] for s in strategy_objs])) else: - _print_objs_tabular(strategy_objs) + _print_objs_tabular(strategy_objs, config.get('print_colorized', False)) def start_list_hyperopts(args: Dict[str, Any]) -> None: @@ -82,7 +93,7 @@ def start_list_hyperopts(args: Dict[str, Any]) -> None: if args['print_one_column']: print('\n'.join([s['name'] for s in hyperopt_objs])) else: - _print_objs_tabular(hyperopt_objs) + _print_objs_tabular(hyperopt_objs, config.get('print_colorized', False)) def start_list_timeframes(args: Dict[str, Any]) -> None: From 47a91c9d8ec0706add136b3b1713c586fadf4b5e Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 22:32:46 +0300 Subject: [PATCH 248/317] Remove green color --- freqtrade/commands/list_commands.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 6a2ccbfcf..67ee59375 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -49,9 +49,7 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: 'status': (((Fore.RED if print_colorized else '') + "LOAD FAILED" + (Style.RESET_ALL if print_colorized else '')) if s['class'] is None - else ((Fore.GREEN if print_colorized else '') + - "OK" + (Style.RESET_ALL if print_colorized else '')) - if names.count(s['name']) == 1 + else "OK" if names.count(s['name']) == 1 else ((Fore.YELLOW if print_colorized else '') + "DUPLICATED NAME" + (Style.RESET_ALL if print_colorized else ''))) From 06b84b4086d6807bb6ac861973028e799dcaea91 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Feb 2020 23:13:49 +0300 Subject: [PATCH 249/317] Remove redundant code --- freqtrade/resolvers/iresolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index d674daa9a..84dee85cd 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -164,7 +164,7 @@ class IResolver: enum_failed=enum_failed): objects.append( {'name': obj.__name__ if obj is not None else '', - 'class': obj if obj is not None else None, + 'class': obj, 'location': entry, }) return objects From 93f9ff1b636724ad729a88b0dc4cb35fa6612e35 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 15 Feb 2020 04:22:21 +0300 Subject: [PATCH 250/317] Fix existing test --- tests/strategy/test_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index d3977ae44..1b4b64f89 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -32,7 +32,7 @@ def test_search_strategy(): def test_search_all_strategies(): directory = Path(__file__).parent - strategies = StrategyResolver.search_all_objects(directory) + strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) assert len(strategies) == 3 assert isinstance(strategies[0], dict) From 29d9b6a46a94ed222a97bb1e14081c1948f2cca1 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 15 Feb 2020 04:32:10 +0300 Subject: [PATCH 251/317] Add test for enum failed --- tests/strategy/test_strategy.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 1b4b64f89..3d3f0f424 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -30,7 +30,7 @@ def test_search_strategy(): assert s is None -def test_search_all_strategies(): +def test_search_all_strategies_no_failed(): directory = Path(__file__).parent strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) @@ -38,6 +38,15 @@ def test_search_all_strategies(): assert isinstance(strategies[0], dict) +def test_search_all_strategies_with_failed(): + directory = Path(__file__).parent + strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) + assert isinstance(strategies, list) + assert len(strategies) == 4 + assert isinstance(strategies[0], dict) + assert strategies[0]['class'] is None + + def test_load_strategy(default_conf, result): default_conf.update({'strategy': 'SampleStrategy', 'strategy_path': str(Path(__file__).parents[2] / 'freqtrade/templates') From 1cf19133f47237fa42e693e96bbdc42f072740ca Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 15 Feb 2020 05:17:34 +0300 Subject: [PATCH 252/317] Added missing failing strategy --- tests/strategy/failing_strategy.py | 87 ++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/strategy/failing_strategy.py diff --git a/tests/strategy/failing_strategy.py b/tests/strategy/failing_strategy.py new file mode 100644 index 000000000..57a8cc1ae --- /dev/null +++ b/tests/strategy/failing_strategy.py @@ -0,0 +1,87 @@ + +# --- Do not remove these libs --- +from freqtrade.strategy.interface import IStrategy +from pandas import DataFrame +# -------------------------------- + +# Add your lib to import here +import talib.abstract as ta + +import nonexiting_module # noqa + + +# This class is a sample. Feel free to customize it. +class TestStrategyLegacy(IStrategy): + """ + This is a test strategy using the legacy function headers, which will be + removed in a future update. + Please do not use this as a template, but refer to user_data/strategy/sample_strategy.py + for a uptodate version of this template. + """ + + # Minimal ROI designed for the strategy. + # This attribute will be overridden if the config file contains "minimal_roi" + minimal_roi = { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 + } + + # Optimal stoploss designed for the strategy + # This attribute will be overridden if the config file contains "stoploss" + stoploss = -0.10 + + # Optimal ticker interval for the strategy + ticker_interval = '5m' + + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + + Performance Note: For the best performance be frugal on the number of indicators + you are using. Let uncomment only the indicator you are using in your strategies + or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + """ + + # Momentum Indicator + # ------------------------------------ + + # ADX + dataframe['adx'] = ta.ADX(dataframe) + + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + + return dataframe + + def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + """ + Based on TA indicators, populates the buy signal for the given dataframe + :param dataframe: DataFrame + :return: DataFrame with buy column + """ + dataframe.loc[ + ( + (dataframe['adx'] > 30) & + (dataframe['tema'] > dataframe['tema'].shift(1)) & + (dataframe['volume'] > 0) + ), + 'buy'] = 1 + + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + """ + Based on TA indicators, populates the sell signal for the given dataframe + :param dataframe: DataFrame + :return: DataFrame with buy column + """ + dataframe.loc[ + ( + (dataframe['adx'] > 70) & + (dataframe['tema'] < dataframe['tema'].shift(1)) & + (dataframe['volume'] > 0) + ), + 'sell'] = 1 + return dataframe From e8c0a0bcd3b242e49f02a620d489cf352b548aa3 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 15 Feb 2020 06:18:00 +0300 Subject: [PATCH 253/317] Make mypy happy --- freqtrade/resolvers/iresolver.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 84dee85cd..34f3934b6 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -7,7 +7,7 @@ import importlib.util import inspect import logging from pathlib import Path -from typing import Any, Dict, Generator, List, Optional, Tuple, Type, Union +from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union from freqtrade.exceptions import OperationalException @@ -40,10 +40,8 @@ class IResolver: return abs_paths @classmethod - def _get_valid_object(cls, module_path: Path, - object_name: Optional[str], - enum_failed: bool = False) -> Union[Generator[Any, None, None], - Tuple[None]]: + def _get_valid_object(cls, module_path: Path, object_name: Optional[str], + enum_failed: bool = False) -> Iterator[Any]: """ Generator returning objects with matching object_type and object_name in the path given. :param module_path: absolute path to the module @@ -63,7 +61,7 @@ class IResolver: # Catch errors in case a specific module is not installed logger.warning(f"Could not import {module_path} due to '{err}'") if enum_failed: - return (None, ) + return iter([None]) valid_objects_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) From ddea4b9300f12fcd099f0e4f5724bf290ec17881 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 15 Feb 2020 06:54:18 +0300 Subject: [PATCH 254/317] Fix test --- tests/strategy/test_strategy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 3d3f0f424..379260599 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -43,8 +43,10 @@ def test_search_all_strategies_with_failed(): strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) assert len(strategies) == 4 - assert isinstance(strategies[0], dict) - assert strategies[0]['class'] is None + # with enum_failed=True search_all_objects() shall find 3 good strategies + # and 1 which fails to load + assert len([x for x in strategies if x['class'] is not None]) == 3 + assert len([x for x in strategies if x['class'] is None]) == 1 def test_load_strategy(default_conf, result): From 42a5d78e607f226e4e65cf5157f65bdf28ec54f5 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 15 Feb 2020 07:19:24 +0300 Subject: [PATCH 255/317] Wording (duplicate, not duplicated) --- freqtrade/commands/list_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 67ee59375..0cfc78596 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -51,7 +51,7 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: if s['class'] is None else "OK" if names.count(s['name']) == 1 else ((Fore.YELLOW if print_colorized else '') + - "DUPLICATED NAME" + + "DUPLICATE NAME" + (Style.RESET_ALL if print_colorized else ''))) } for s in objs] From fdd362299f5c7018f500b626f884f7dd27c17e62 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 15 Feb 2020 07:34:39 +0300 Subject: [PATCH 256/317] Docs adjusted --- docs/utils.md | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index abb7fd0db..cdf0c31af 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -144,38 +144,47 @@ freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt Use the `list-strategies` subcommand to see all strategies in one particular directory and the `list-hyperopts` subcommand to list custom Hyperopts. +These subcommands are useful for finding problems in your environment with loading strategies or hyperopt classes: modules with strategies or hyperopt classes that contain errors and failed to load are printed in red (LOAD FAILED), while strategies or hyperopt classes with duplicate names are printed in yellow (DUPLICATE NAME). + ``` -freqtrade list-strategies --help -usage: freqtrade list-strategies [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--strategy-path PATH] [-1] +usage: freqtrade list-strategies [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [--strategy-path PATH] [-1] [--no-color] optional arguments: -h, --help show this help message and exit --strategy-path PATH Specify additional strategy lookup path. -1, --one-column Print output in one column. + --no-color Disable colorization of hyperopt results. May be + useful if you are redirecting output to a file. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` - to read config from stdin. + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH Path to userdata directory. ``` ``` -freqtrade list-hyperopts --help usage: freqtrade list-hyperopts [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] - [--hyperopt-path PATH] [-1] + [--hyperopt-path PATH] [-1] [--no-color] optional arguments: -h, --help show this help message and exit --hyperopt-path PATH Specify additional lookup path for Hyperopt and Hyperopt Loss functions. -1, --one-column Print output in one column. + --no-color Disable colorization of hyperopt results. May be + useful if you are redirecting output to a file. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). From 87b506972f5b99f11e4f5c97a54f69c716283558 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Feb 2020 13:12:29 +0100 Subject: [PATCH 257/317] Fix edge documentation rendering --- docs/edge.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index dcefe7451..6a301b044 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -145,19 +145,19 @@ Edge module has following configuration options: | Parameter | Description | |------------|-------------| -| `enabled` | If true, then Edge will run periodically.
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `process_throttle_secs` | How often should Edge run in seconds.
*Defaults to `3600` (once per hour).*
***Datatype:*** *Integer* -| `calculate_since_number_of_days` | Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy.
**Note** that it downloads historical data so increasing this number would lead to slowing down the bot.
*Defaults to `7`.*
***Datatype:*** *Integer* -| `capital_available_percentage` | **DEPRECATED - [replaced with `tradable_balance_ratio`](configuration.md#Available balance)** This is the percentage of the total capital on exchange in stake currency.
As an example if you have 10 ETH available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers it as available capital.
*Defaults to `0.5`.*
***Datatype:*** *Float* -| `allowed_risk` | Ratio of allowed risk per trade.
*Defaults to `0.01` (1%)).*
***Datatype:*** *Float* -| `stoploss_range_min` | Minimum stoploss.
*Defaults to `-0.01`.*
***Datatype:*** *Float* -| `stoploss_range_max` | Maximum stoploss.
*Defaults to `-0.10`.*
***Datatype:*** *Float* -| `stoploss_range_step` | As an example if this is set to -0.01 then Edge will test the strategy for `[-0.01, -0,02, -0,03 ..., -0.09, -0.10]` ranges.
**Note** than having a smaller step means having a bigger range which could lead to slow calculation.
If you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10.
*Defaults to `-0.001`.*
***Datatype:*** *Float* -| `minimum_winrate` | It filters out pairs which don't have at least minimum_winrate.
This comes handy if you want to be conservative and don't comprise win rate in favour of risk reward ratio.
*Defaults to `0.60`.*
***Datatype:*** *Float* -| `minimum_expectancy` | It filters out pairs which have the expectancy lower than this number.
Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return.
*Defaults to `0.20`.*
***Datatype:*** *Float* -| `min_trade_number` | When calculating *W*, *R* and *E* (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable.
Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
*Defaults to `10` (it is highly recommended not to decrease this number).*
***Datatype:*** *Integer* -| `max_trade_duration_minute` | Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
**NOTICE:** While configuring this value, you should take into consideration your ticker interval. As an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. Default value is set assuming your strategy interval is relatively small (1m or 5m, etc.).
*Defaults to `1440` (one day).*
***Datatype:*** *Integer* -| `remove_pumps` | Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
*Defaults to `false`.*
***Datatype:*** *Boolean* +| `enabled` | If true, then Edge will run periodically.
*Defaults to `false`.*
**Datatype:** Boolean +| `process_throttle_secs` | How often should Edge run in seconds.
*Defaults to `3600` (once per hour).*
**Datatype:** Integer +| `calculate_since_number_of_days` | Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy.
**Note** that it downloads historical data so increasing this number would lead to slowing down the bot.
*Defaults to `7`.*
**Datatype:** Integer +| `capital_available_percentage` | **DEPRECATED - [replaced with `tradable_balance_ratio`](configuration.md#Available balance)** This is the percentage of the total capital on exchange in stake currency.
As an example if you have 10 ETH available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers it as available capital.
*Defaults to `0.5`.*
**Datatype:** Float +| `allowed_risk` | Ratio of allowed risk per trade.
*Defaults to `0.01` (1%)).*
**Datatype:** Float +| `stoploss_range_min` | Minimum stoploss.
*Defaults to `-0.01`.*
**Datatype:** Float +| `stoploss_range_max` | Maximum stoploss.
*Defaults to `-0.10`.*
**Datatype:** Float +| `stoploss_range_step` | As an example if this is set to -0.01 then Edge will test the strategy for `[-0.01, -0,02, -0,03 ..., -0.09, -0.10]` ranges.
**Note** than having a smaller step means having a bigger range which could lead to slow calculation.
If you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10.
*Defaults to `-0.001`.*
**Datatype:** Float +| `minimum_winrate` | It filters out pairs which don't have at least minimum_winrate.
This comes handy if you want to be conservative and don't comprise win rate in favour of risk reward ratio.
*Defaults to `0.60`.*
**Datatype:** Float +| `minimum_expectancy` | It filters out pairs which have the expectancy lower than this number.
Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return.
*Defaults to `0.20`.*
**Datatype:** Float +| `min_trade_number` | When calculating *W*, *R* and *E* (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable.
Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
*Defaults to `10` (it is highly recommended not to decrease this number).*
**Datatype:** Integer +| `max_trade_duration_minute` | Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
**NOTICE:** While configuring this value, you should take into consideration your ticker interval. As an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. Default value is set assuming your strategy interval is relatively small (1m or 5m, etc.).
*Defaults to `1440` (one day).*
**Datatype:** Integer +| `remove_pumps` | Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
*Defaults to `false`.*
**Datatype:** Boolean ## Running Edge independently From 6139239b863b36522a7d96efdb859e830403bad9 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 15 Feb 2020 20:43:11 +0300 Subject: [PATCH 258/317] Address points stated in comments --- freqtrade/commands/list_commands.py | 15 +++-- freqtrade/resolvers/iresolver.py | 2 +- tests/strategy/failing_strategy.py | 86 ++--------------------------- 3 files changed, 14 insertions(+), 89 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 0cfc78596..49674b81a 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -41,18 +41,21 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: def _print_objs_tabular(objs: List, print_colorized: bool) -> None: if print_colorized: colorama_init(autoreset=True) + red = Fore.RED + yellow = Fore.YELLOW + reset = Style.RESET_ALL + else: + red = '' + yellow = '' + reset = '' names = [s['name'] for s in objs] objss_to_print = [{ 'name': s['name'] if s['name'] else "--", 'location': s['location'].name, - 'status': (((Fore.RED if print_colorized else '') + - "LOAD FAILED" + (Style.RESET_ALL if print_colorized else '')) - if s['class'] is None + 'status': (red + "LOAD FAILED" + reset if s['class'] is None else "OK" if names.count(s['name']) == 1 - else ((Fore.YELLOW if print_colorized else '') + - "DUPLICATE NAME" + - (Style.RESET_ALL if print_colorized else ''))) + else yellow + "DUPLICATE NAME" + reset) } for s in objs] print(tabulate(objss_to_print, headers='keys', tablefmt='pipe')) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 34f3934b6..922a2700a 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -87,7 +87,7 @@ class IResolver: continue module_path = entry.resolve() - obj = next(cls._get_valid_object(module_path, object_name), None) # noqa + obj = next(cls._get_valid_object(module_path, object_name), None) if obj: return (obj, module_path) diff --git a/tests/strategy/failing_strategy.py b/tests/strategy/failing_strategy.py index 57a8cc1ae..f8eaac3c3 100644 --- a/tests/strategy/failing_strategy.py +++ b/tests/strategy/failing_strategy.py @@ -1,87 +1,9 @@ - -# --- Do not remove these libs --- -from freqtrade.strategy.interface import IStrategy -from pandas import DataFrame -# -------------------------------- - -# Add your lib to import here -import talib.abstract as ta +# The strategy which fails to load due to non-existent dependency import nonexiting_module # noqa +from freqtrade.strategy.interface import IStrategy + -# This class is a sample. Feel free to customize it. class TestStrategyLegacy(IStrategy): - """ - This is a test strategy using the legacy function headers, which will be - removed in a future update. - Please do not use this as a template, but refer to user_data/strategy/sample_strategy.py - for a uptodate version of this template. - """ - - # Minimal ROI designed for the strategy. - # This attribute will be overridden if the config file contains "minimal_roi" - minimal_roi = { - "40": 0.0, - "30": 0.01, - "20": 0.02, - "0": 0.04 - } - - # Optimal stoploss designed for the strategy - # This attribute will be overridden if the config file contains "stoploss" - stoploss = -0.10 - - # Optimal ticker interval for the strategy - ticker_interval = '5m' - - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - - Performance Note: For the best performance be frugal on the number of indicators - you are using. Let uncomment only the indicator you are using in your strategies - or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - """ - - # Momentum Indicator - # ------------------------------------ - - # ADX - dataframe['adx'] = ta.ADX(dataframe) - - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - - return dataframe - - def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - dataframe.loc[ - ( - (dataframe['adx'] > 30) & - (dataframe['tema'] > dataframe['tema'].shift(1)) & - (dataframe['volume'] > 0) - ), - 'buy'] = 1 - - return dataframe - - def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - dataframe.loc[ - ( - (dataframe['adx'] > 70) & - (dataframe['tema'] < dataframe['tema'].shift(1)) & - (dataframe['volume'] > 0) - ), - 'sell'] = 1 - return dataframe + pass From 6e71f2f1662246b848d8137223c8449068f2348e Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 15 Feb 2020 20:55:12 +0100 Subject: [PATCH 259/317] my fix --- freqtrade/rpc/rpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c182aad2b..f6ac999f7 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -168,9 +168,9 @@ class RPC: profit_str += f" ({fiat_profit:.2f})" trades_list.append([ trade.id, - trade.pair + '*' if (trade.open_order_id is not None - and trade.close_rate_requested is None) else '' - + '**' if (trade.close_rate_requested is not None) else '', + trade.pair + ['', '*'][trade.open_order_id is not None + and trade.close_rate_requested is None] + + ['', '**'][trade.close_rate_requested is not None], shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), profit_str ]) From 180939a962e1214b744999c449753be26024edad Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 15 Feb 2020 21:01:45 +0100 Subject: [PATCH 260/317] winner, readability, with brackets as fix --- freqtrade/rpc/rpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f6ac999f7..3411318bb 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -168,9 +168,9 @@ class RPC: profit_str += f" ({fiat_profit:.2f})" trades_list.append([ trade.id, - trade.pair + ['', '*'][trade.open_order_id is not None - and trade.close_rate_requested is None] - + ['', '**'][trade.close_rate_requested is not None], + trade.pair + ('*' if (trade.open_order_id is not None + and trade.close_rate_requested is None) else '') + + ('**' if (trade.close_rate_requested is not None) else ''), shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), profit_str ]) From 3787ac7b980e16d615cebe5a942a8d45490ba72c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Feb 2020 13:20:11 +0100 Subject: [PATCH 261/317] increment limit to adjust to FTX defaults (1500 candles) --- freqtrade/exchange/__init__.py | 18 ++++++++++-------- freqtrade/exchange/ftx.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 freqtrade/exchange/ftx.py diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index df18bca02..a39f8f5df 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,18 +1,20 @@ -from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS # noqa: F401 -from freqtrade.exchange.exchange import Exchange # noqa: F401 -from freqtrade.exchange.exchange import (get_exchange_bad_reason, # noqa: F401 +# flake8: noqa: F401 +from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS +from freqtrade.exchange.exchange import Exchange +from freqtrade.exchange.exchange import (get_exchange_bad_reason, is_exchange_bad, is_exchange_known_ccxt, is_exchange_officially_supported, ccxt_exchanges, available_exchanges) -from freqtrade.exchange.exchange import (timeframe_to_seconds, # noqa: F401 +from freqtrade.exchange.exchange import (timeframe_to_seconds, timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date) -from freqtrade.exchange.exchange import (market_is_active, # noqa: F401 +from freqtrade.exchange.exchange import (market_is_active, symbol_is_pair) -from freqtrade.exchange.kraken import Kraken # noqa: F401 -from freqtrade.exchange.binance import Binance # noqa: F401 -from freqtrade.exchange.bibox import Bibox # noqa: F401 +from freqtrade.exchange.kraken import Kraken +from freqtrade.exchange.binance import Binance +from freqtrade.exchange.bibox import Bibox +from freqtrade.exchange.ftx import Ftx diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py new file mode 100644 index 000000000..75915122b --- /dev/null +++ b/freqtrade/exchange/ftx.py @@ -0,0 +1,14 @@ +""" FTX exchange subclass """ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + +logger = logging.getLogger(__name__) + + +class Ftx(Exchange): + + _ft_has: Dict = { + "ohlcv_candle_limit": 1500, + } From bec86b13258f97dba75b7b5b8cc39f773490d7fa Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Feb 2020 15:42:41 +0100 Subject: [PATCH 262/317] Add github actions badge --- README.md | 2 +- docs/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a1feeab67..59799da84 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Freqtrade -[![Build Status](https://travis-ci.org/freqtrade/freqtrade.svg?branch=develop)](https://travis-ci.org/freqtrade/freqtrade) +[![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/) [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop) [![Documentation](https://readthedocs.org/projects/freqtrade/badge/)](https://www.freqtrade.io) [![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability) diff --git a/docs/index.md b/docs/index.md index f0ee831e3..adc661300 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,5 @@ # Freqtrade -[![Build Status](https://travis-ci.org/freqtrade/freqtrade.svg?branch=develop)](https://travis-ci.org/freqtrade/freqtrade) +[![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/) [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop) [![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability) From 212d20ed087161cc4abefdbe47a0c4c345e955e7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2020 08:03:37 +0000 Subject: [PATCH 263/317] Bump ccxt from 1.22.39 to 1.22.61 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.22.39 to 1.22.61. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.22.39...1.22.61) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index f641dd2ad..e89afceed 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.22.39 +ccxt==1.22.61 SQLAlchemy==1.3.13 python-telegram-bot==12.4.1 arrow==0.15.5 From c6a3038f5274fe974fd34e51b313689ce5112c52 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2020 08:03:57 +0000 Subject: [PATCH 264/317] Bump coveralls from 1.10.0 to 1.11.1 Bumps [coveralls](https://github.com/coveralls-clients/coveralls-python) from 1.10.0 to 1.11.1. - [Release notes](https://github.com/coveralls-clients/coveralls-python/releases) - [Changelog](https://github.com/coveralls-clients/coveralls-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/coveralls-clients/coveralls-python/compare/1.10.0...1.11.1) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 268c5f777..1e58ae6e0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,7 @@ -r requirements-plot.txt -r requirements-hyperopt.txt -coveralls==1.10.0 +coveralls==1.11.1 flake8==3.7.9 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.0.0 From 500e1c77de3912c989612fd2a823c5c72a3a8415 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2020 08:04:21 +0000 Subject: [PATCH 265/317] Bump wrapt from 1.11.2 to 1.12.0 Bumps [wrapt](https://github.com/GrahamDumpleton/wrapt) from 1.11.2 to 1.12.0. - [Release notes](https://github.com/GrahamDumpleton/wrapt/releases) - [Changelog](https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst) - [Commits](https://github.com/GrahamDumpleton/wrapt/compare/1.11.2...1.12.0) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index f641dd2ad..df67c1691 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -7,7 +7,7 @@ arrow==0.15.5 cachetools==4.0.0 requests==2.22.0 urllib3==1.25.8 -wrapt==1.11.2 +wrapt==1.12.0 jsonschema==3.2.0 TA-Lib==0.4.17 tabulate==0.8.6 From 9435950fc968899d697263244096872cded9951c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2020 08:04:40 +0000 Subject: [PATCH 266/317] Bump mkdocs-material from 4.6.2 to 4.6.3 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 4.6.2 to 4.6.3. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/4.6.2...4.6.3) Signed-off-by: dependabot-preview[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 3980ecd64..48ade026e 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,2 +1,2 @@ -mkdocs-material==4.6.2 +mkdocs-material==4.6.3 mdx_truly_sane_lists==1.2 From 0fd3d74fc4ab745833b682cc69a6eb3581de30da Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2020 08:05:00 +0000 Subject: [PATCH 267/317] Bump scikit-optimize from 0.7.1 to 0.7.2 Bumps [scikit-optimize](https://github.com/scikit-optimize/scikit-optimize) from 0.7.1 to 0.7.2. - [Release notes](https://github.com/scikit-optimize/scikit-optimize/releases) - [Changelog](https://github.com/scikit-optimize/scikit-optimize/blob/master/CHANGELOG.md) - [Commits](https://github.com/scikit-optimize/scikit-optimize/compare/v0.7.1...v0.7.2) Signed-off-by: dependabot-preview[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 202806cef..e97e7f6be 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -4,6 +4,6 @@ # Required for hyperopt scipy==1.4.1 scikit-learn==0.22.1 -scikit-optimize==0.7.1 +scikit-optimize==0.7.2 filelock==3.0.12 joblib==0.14.1 From 582b59044c3d6388ca1847643c8816b9613156e3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2020 08:47:39 +0000 Subject: [PATCH 268/317] Bump python-telegram-bot from 12.4.1 to 12.4.2 Bumps [python-telegram-bot](https://github.com/python-telegram-bot/python-telegram-bot) from 12.4.1 to 12.4.2. - [Release notes](https://github.com/python-telegram-bot/python-telegram-bot/releases) - [Changelog](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/CHANGES.rst) - [Commits](https://github.com/python-telegram-bot/python-telegram-bot/compare/v12.4.1...v12.4.2) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index e89afceed..6cc8e3809 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -2,7 +2,7 @@ # mainly used for Raspberry pi installs ccxt==1.22.61 SQLAlchemy==1.3.13 -python-telegram-bot==12.4.1 +python-telegram-bot==12.4.2 arrow==0.15.5 cachetools==4.0.0 requests==2.22.0 From 0b33b798e4a9d5e6f13df7a3d3a0e90dbe34643e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Feb 2020 20:16:24 +0100 Subject: [PATCH 269/317] Add pypi build step --- .github/workflows/ci.yml | 29 ++++++++++++++++++++++++++++- freqtrade/__init__.py | 2 +- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05d151a88..cc8906af5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,8 @@ on: - develop - github_actions_tests tags: + release: + types: [published] pull_request: schedule: - cron: '0 5 * * 4' @@ -191,15 +193,40 @@ jobs: deploy: needs: [ build, build_windows, docs_check ] runs-on: ubuntu-18.04 - if: (github.event_name == 'push' || github.event_name == 'schedule') && github.repository == 'freqtrade/freqtrade' + if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' steps: - uses: actions/checkout@v1 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Extract branch name shell: bash run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" id: extract_branch + - name: Build distribution + run: | + pip install -U setuptools wheel + python setup.py sdist bdist_wheel + + - name: Publish to PyPI (Test) + uses: pypa/gh-action-pypi-publish@master + if: (steps.extract_branch.outputs.branch == 'master' || github.event_name == 'release') + with: + user: __token__ + password: ${{ secrets.pypi_test_password }} + repository_url: https://test.pypi.org/legacy/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@master + if: (steps.extract_branch.outputs.branch == 'master' || github.event_name == 'release') + with: + user: __token__ + password: ${{ secrets.pypi_password }} + - name: Build and test and push docker image env: IMAGE_NAME: freqtradeorg/freqtrade diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index e1f65d4fe..f2ae5dc63 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,4 +1,4 @@ -""" FreqTrade bot """ +""" Freqtrade bot """ __version__ = 'develop' if __version__ == 'develop': From 1172c958174424e138d9391dfcd90db6e6856705 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Feb 2020 20:17:08 +0100 Subject: [PATCH 270/317] Use different versioning scheme --- freqtrade/__init__.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index f2ae5dc63..23ab945b4 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -5,9 +5,23 @@ if __version__ == 'develop': try: import subprocess - __version__ = 'develop-' + subprocess.check_output( - ['git', 'log', '--format="%h"', '-n 1'], - stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') + + # __version__ = 'develop-' + subprocess.check_output( + # ['git', 'log', '--format="%h"', '-n 1'], + # stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') + + from datetime import datetime + last_release = subprocess.check_output( + ['git', 'tag'] + ).decode('utf-8').split()[-1].split(".") + # Releases are in the format "2020.1" - we increment the latest version for dev. + prefix = f"{last_release[0]}.{int(last_release[1]) + 1}" + dev_version = int(datetime.now().timestamp() // 1000) + __version__ = f"{prefix}.dev{dev_version}" + + # subprocess.check_output( + # ['git', 'log', '--format="%h"', '-n 1'], + # stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') except Exception: # git not available, ignore pass From e6dd463ca3ad7803e1e6df391d3917d32142368e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Feb 2020 20:17:36 +0100 Subject: [PATCH 271/317] Revert versioning --- freqtrade/__init__.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 23ab945b4..ad432a20b 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -6,18 +6,18 @@ if __version__ == 'develop': try: import subprocess - # __version__ = 'develop-' + subprocess.check_output( - # ['git', 'log', '--format="%h"', '-n 1'], - # stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') + __version__ = 'develop-' + subprocess.check_output( + ['git', 'log', '--format="%h"', '-n 1'], + stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') - from datetime import datetime - last_release = subprocess.check_output( - ['git', 'tag'] - ).decode('utf-8').split()[-1].split(".") - # Releases are in the format "2020.1" - we increment the latest version for dev. - prefix = f"{last_release[0]}.{int(last_release[1]) + 1}" - dev_version = int(datetime.now().timestamp() // 1000) - __version__ = f"{prefix}.dev{dev_version}" + # from datetime import datetime + # last_release = subprocess.check_output( + # ['git', 'tag'] + # ).decode('utf-8').split()[-1].split(".") + # # Releases are in the format "2020.1" - we increment the latest version for dev. + # prefix = f"{last_release[0]}.{int(last_release[1]) + 1}" + # dev_version = int(datetime.now().timestamp() // 1000) + # __version__ = f"{prefix}.dev{dev_version}" # subprocess.check_output( # ['git', 'log', '--format="%h"', '-n 1'], From 1634297685489bd1dd47802801bef7daae1a7238 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 18 Feb 2020 20:12:10 +0100 Subject: [PATCH 272/317] Move strategies to test subfolder --- tests/commands/test_commands.py | 4 ++-- tests/conftest.py | 1 + .../strategy/strats}/default_strategy.py | 0 tests/strategy/{ => strats}/failing_strategy.py | 0 tests/strategy/{ => strats}/legacy_strategy.py | 0 tests/strategy/test_default_strategy.py | 2 +- tests/strategy/test_interface.py | 2 +- tests/strategy/test_strategy.py | 14 +++++++------- 8 files changed, 12 insertions(+), 11 deletions(-) rename {freqtrade/strategy => tests/strategy/strats}/default_strategy.py (100%) rename tests/strategy/{ => strats}/failing_strategy.py (100%) rename tests/strategy/{ => strats}/legacy_strategy.py (100%) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index ee1db5db5..55bd7306d 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -640,7 +640,7 @@ def test_start_list_strategies(mocker, caplog, capsys): args = [ "list-strategies", "--strategy-path", - str(Path(__file__).parent.parent / "strategy"), + str(Path(__file__).parent.parent / "strategy" / "strats"), "-1" ] pargs = get_args(args) @@ -655,7 +655,7 @@ def test_start_list_strategies(mocker, caplog, capsys): args = [ "list-strategies", "--strategy-path", - str(Path(__file__).parent.parent / "strategy"), + str(Path(__file__).parent.parent / "strategy" / "strats"), ] pargs = get_args(args) # pargs['config'] = None diff --git a/tests/conftest.py b/tests/conftest.py index e897dbccd..acb730330 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -257,6 +257,7 @@ def default_conf(testdatadir): "db_url": "sqlite://", "user_data_dir": Path("user_data"), "verbosity": 3, + "strategy_path": str(Path(__file__).parent / "strategy" / "strats"), "strategy": "DefaultStrategy" } return configuration diff --git a/freqtrade/strategy/default_strategy.py b/tests/strategy/strats/default_strategy.py similarity index 100% rename from freqtrade/strategy/default_strategy.py rename to tests/strategy/strats/default_strategy.py diff --git a/tests/strategy/failing_strategy.py b/tests/strategy/strats/failing_strategy.py similarity index 100% rename from tests/strategy/failing_strategy.py rename to tests/strategy/strats/failing_strategy.py diff --git a/tests/strategy/legacy_strategy.py b/tests/strategy/strats/legacy_strategy.py similarity index 100% rename from tests/strategy/legacy_strategy.py rename to tests/strategy/strats/legacy_strategy.py diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index 17d6b8ee0..0b8ea9f85 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -1,6 +1,6 @@ from pandas import DataFrame -from freqtrade.strategy.default_strategy import DefaultStrategy +from .strats.default_strategy import DefaultStrategy def test_default_strategy_structure(): diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index a28519383..def64425e 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -11,7 +11,7 @@ from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.persistence import Trade from freqtrade.resolvers import StrategyResolver -from freqtrade.strategy.default_strategy import DefaultStrategy +from .strats.default_strategy import DefaultStrategy from tests.conftest import get_patched_exchange, log_has # Avoid to reinit the same object again and again diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 379260599..5c6de8260 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -31,21 +31,21 @@ def test_search_strategy(): def test_search_all_strategies_no_failed(): - directory = Path(__file__).parent + directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 3 + assert len(strategies) == 2 assert isinstance(strategies[0], dict) def test_search_all_strategies_with_failed(): - directory = Path(__file__).parent + directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 4 + assert len(strategies) == 3 # with enum_failed=True search_all_objects() shall find 3 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 3 + assert len([x for x in strategies if x['class'] is not None]) == 2 assert len([x for x in strategies if x['class'] is None]) == 1 @@ -326,7 +326,7 @@ def test_strategy_override_use_sell_profit_only(caplog, default_conf): @pytest.mark.filterwarnings("ignore:deprecated") def test_deprecate_populate_indicators(result, default_conf): - default_location = path.join(path.dirname(path.realpath(__file__))) + default_location = Path(__file__).parent / "strats" default_conf.update({'strategy': 'TestStrategyLegacy', 'strategy_path': default_location}) strategy = StrategyResolver.load_strategy(default_conf) @@ -360,7 +360,7 @@ def test_deprecate_populate_indicators(result, default_conf): @pytest.mark.filterwarnings("ignore:deprecated") def test_call_deprecated_function(result, monkeypatch, default_conf): - default_location = path.join(path.dirname(path.realpath(__file__))) + default_location = Path(__file__).parent / "strats" default_conf.update({'strategy': 'TestStrategyLegacy', 'strategy_path': default_location}) strategy = StrategyResolver.load_strategy(default_conf) From d91b9d125314c62119b75142fb833ffd0a829bc1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 18 Feb 2020 20:26:20 +0100 Subject: [PATCH 273/317] Fix some tests, don't default to freqtrade/strategy for imports --- freqtrade/resolvers/iresolver.py | 4 +++- freqtrade/resolvers/strategy_resolver.py | 2 +- tests/strategy/test_strategy.py | 10 ++++------ tests/test_configuration.py | 1 + 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 922a2700a..764759289 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -28,7 +28,9 @@ class IResolver: def build_search_paths(cls, config: Dict[str, Any], user_subdir: Optional[str] = None, extra_dir: Optional[str] = None) -> List[Path]: - abs_paths: List[Path] = [cls.initial_search_path] + abs_paths: List[Path] = [] + if cls.initial_search_path: + abs_paths.append(cls.initial_search_path) if user_subdir: abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir)) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index bb8ff870e..cddc7c9cd 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -27,7 +27,7 @@ class StrategyResolver(IResolver): object_type = IStrategy object_type_str = "Strategy" user_subdir = USERPATH_STRATEGIES - initial_search_path = Path(__file__).parent.parent.joinpath('strategy').resolve() + initial_search_path = None @staticmethod def load_strategy(config: Dict[str, Any] = None) -> IStrategy: diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 5c6de8260..27bbb2d3b 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -2,7 +2,6 @@ import logging import warnings from base64 import urlsafe_b64encode -from os import path from pathlib import Path import pytest @@ -15,7 +14,7 @@ from tests.conftest import log_has, log_has_re def test_search_strategy(): - default_location = Path(__file__).parent.parent.joinpath('strategy').resolve() + default_location = Path(__file__).parent / 'strats' s, _ = StrategyResolver._search_object( directory=default_location, @@ -72,13 +71,12 @@ def test_load_strategy_base64(result, caplog, default_conf): def test_load_strategy_invalid_directory(result, caplog, default_conf): default_conf['strategy'] = 'DefaultStrategy' extra_dir = Path.cwd() / 'some/path' - strategy = StrategyResolver._load_strategy('DefaultStrategy', config=default_conf, - extra_dir=extra_dir) + with pytest.raises(OperationalException): + StrategyResolver._load_strategy('DefaultStrategy', config=default_conf, + extra_dir=extra_dir) assert log_has_re(r'Path .*' + r'some.*path.*' + r'.* does not exist', caplog) - assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) - def test_load_not_found_strategy(default_conf): default_conf['strategy'] = 'NotFoundStrategy' diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 74de166c1..d810305db 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -212,6 +212,7 @@ def test_load_config_file_exception(mocker) -> None: def test_load_config(default_conf, mocker) -> None: + del default_conf['strategy_path'] patched_configuration_load_config_file(mocker, default_conf) args = Arguments(['trade']).get_parsed_arg() From 09d89fbfb39b5aa5f75a589b82bdba11800656b0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Feb 2020 07:15:55 +0100 Subject: [PATCH 274/317] Fix last test --- tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index bba15c156..9cfd662c1 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -759,7 +759,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'backtesting', '--config', 'config.json', '--datadir', str(testdatadir), - '--strategy-path', str(Path(__file__).parents[2] / 'freqtrade/templates'), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), '--ticker-interval', '1m', '--timerange', '1510694220-1510700340', '--enable-position-stacking', From 882d0a59330d81c65ca79bc93270247d9cd439ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Feb 2020 12:54:59 +0100 Subject: [PATCH 275/317] implement documentation feedback after review --- docs/data-download.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index f747123d0..a01aaf4f6 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -52,8 +52,8 @@ Common arguments: ### Data format -Freqtrade currently supports 2 dataformats, `json` and `jsongz`, a zipped version of json files. -By default, OHLCV data is stored as json data, while trades data is stored as `jsongz` data. +Freqtrade currently supports 2 dataformats, `json` (plain "text" json files) and `jsongz` (a gzipped version of json files). +By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. This can be changed via the `--data-format` and `--data-format-trades` parameters respectivly. @@ -105,8 +105,8 @@ Common arguments: ##### Example converting data -The following command will convert all Candle data available in `~/.freqtrade/data/binance` from json to jsongz, saving diskspace in the process. -It'll also remove source files (`--erase` parameter). +The following command will convert all ohlcv (candle) data available in `~/.freqtrade/data/binance` from json to jsongz, saving diskspace in the process. +It'll also remove original json data files (`--erase` parameter). ``` bash freqtrade convert-data --format-from json --format-to jsongz --data-dir ~/.freqtrade/data/binance -t 5m 15m --erase @@ -151,8 +151,8 @@ Common arguments: ##### Example converting trades -The following command will convert all available trade-data in `~/.freqtrade/data/kraken` from json to jsongz, saving diskspace in the process. -It'll also remove source files (`--erase` parameter). +The following command will convert all available trade-data in `~/.freqtrade/data/kraken` from jsongz to json. +It'll also remove original jsongz data files (`--erase` parameter). ``` bash freqtrade convert-trade-data --format-from jsongz --format-to json --data-dir ~/.freqtrade/data/kraken --erase From 29b369c65ed034f8dadbf6e237a0cefbeb11b7f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Feb 2020 14:53:54 +0100 Subject: [PATCH 276/317] Rename cli argument --- docs/data-download.md | 6 +++--- freqtrade/commands/cli_options.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index a01aaf4f6..76e22f4ea 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -18,7 +18,7 @@ Otherwise `--exchange` becomes mandatory. usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] [--pairs-file FILE] [--days INT] [--dl-trades] [--exchange EXCHANGE] [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]] - [--erase] [--data-format {json,jsongz}] [--data-format-trades {json,jsongz}] + [--erase] [--data-format-ohlcv {json,jsongz}] [--data-format-trades {json,jsongz}] optional arguments: -h, --help show this help message and exit @@ -32,7 +32,7 @@ optional arguments: -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...] Specify which tickers to download. Space-separated list. Default: `1m 5m`. --erase Clean all existing data for the selected exchange/pairs/timeframes. - --data-format {json,jsongz} + --data-format-ohlcv {json,jsongz} Storage format for downloaded ohlcv data. (default: `json`). --data-format-trades {json,jsongz} Storage format for downloaded trades data. (default: `jsongz`). @@ -55,7 +55,7 @@ Common arguments: Freqtrade currently supports 2 dataformats, `json` (plain "text" json files) and `jsongz` (a gzipped version of json files). By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. -This can be changed via the `--data-format` and `--data-format-trades` parameters respectivly. +This can be changed via the `--data-format-ohlcv` and `--data-format-trades` parameters respectivly. If the default dataformat has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well. diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 58cb1e83b..a8d4bc198 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -346,7 +346,7 @@ AVAILABLE_CLI_OPTIONS = { required=True, ), "dataformat_ohlcv": Arg( - '--data-format', + '--data-format-ohlcv', help='Storage format for downloaded ohlcv data. (default: `%(default)s`).', choices=constants.AVAILABLE_DATAHANDLERS, default='json' From d22384c7fba439a5cbe01099dd95844c59934474 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Feb 2020 19:21:48 +0100 Subject: [PATCH 277/317] Full support for kraken stoploss --- freqtrade/persistence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 5b0046091..fa041abc3 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -318,10 +318,10 @@ class Trade(_DECL_BASE): elif order_type in ('market', 'limit') and order['side'] == 'sell': self.close(order['price']) logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self) - elif order_type == 'stop_loss_limit': + elif order_type in ('stop_loss_limit', 'stop-loss'): self.stoploss_order_id = None self.close_rate_requested = self.stop_loss - logger.info('STOP_LOSS_LIMIT is hit for %s.', self) + logger.info('%s is hit for %s.', order_type.upper(), self) self.close(order['average']) else: raise ValueError(f'Unknown order type: {order_type}') From a7342bd9106ac1028fc12b3391c2ab633b9b3919 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Feb 2020 19:42:04 +0100 Subject: [PATCH 278/317] Fix non-existing strategy loading --- tests/optimize/test_backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 9cfd662c1..337194ab1 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -766,7 +766,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): '--disable-max-market-positions', '--strategy-list', 'DefaultStrategy', - 'SampleStrategy', + 'TestStrategyLegacy', ] args = get_args(args) start_backtesting(args) @@ -789,7 +789,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', - 'Running backtesting for Strategy SampleStrategy', + 'Running backtesting for Strategy TestStrategyLegacy', ] for line in exists: From 5adbe3c2d3ac75eb20b08b93d8902cf3ab25a643 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Feb 2020 19:50:01 +0100 Subject: [PATCH 279/317] initial search path is optional ... --- freqtrade/resolvers/iresolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 764759289..52d944f2c 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -22,7 +22,7 @@ class IResolver: object_type: Type[Any] object_type_str: str user_subdir: Optional[str] = None - initial_search_path: Path + initial_search_path: Optional[Path] @classmethod def build_search_paths(cls, config: Dict[str, Any], user_subdir: Optional[str] = None, From bca5f804a87d2b2e1556711a1f96d35161d279a2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 20 Feb 2020 08:17:24 +0300 Subject: [PATCH 280/317] Move divider log message --- freqtrade/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 64cc97026..509ba018e 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -109,14 +109,14 @@ class Worker: """ start = time.time() result = func(*args, **kwargs) + logger.debug("========================================") end = time.time() duration = max(min_secs - (end - start), 0.0) - logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) + logger.debug(f"Throttling {func.__name__} for {duration:.2f} seconds") time.sleep(duration) return result def _process(self) -> None: - logger.debug("========================================") try: self.freqtrade.process() except TemporaryError as error: From 56a06cbd331daa07216b77094d6b98ce0a651439 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 20 Feb 2020 08:19:22 +0300 Subject: [PATCH 281/317] Update strings to f-strings --- freqtrade/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 509ba018e..b3b3b712a 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -26,7 +26,7 @@ class Worker: """ Init all variables and objects the bot needs to work """ - logger.info('Starting worker %s', __version__) + logger.info(f"Starting worker {__version__}") self._args = args self._config = config @@ -77,7 +77,7 @@ class Worker: if state != old_state: self.freqtrade.notify_status(f'{state.name.lower()}') - logger.info('Changing state to: %s', state.name) + logger.info(f"Changing state to: {state.name}") if state == State.RUNNING: self.freqtrade.startup() From 10668bb2490c2ed56f47f5536fdf0892c2173648 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Feb 2020 06:22:36 +0100 Subject: [PATCH 282/317] Update tests/strategy/test_strategy.py Co-Authored-By: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> --- tests/strategy/test_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 27bbb2d3b..13ca68bf0 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -42,7 +42,7 @@ def test_search_all_strategies_with_failed(): strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) assert len(strategies) == 3 - # with enum_failed=True search_all_objects() shall find 3 good strategies + # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load assert len([x for x in strategies if x['class'] is not None]) == 2 assert len([x for x in strategies if x['class'] is None]) == 1 From 78ee36a8c6e8d17289059be945324c8166d514dd Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 20 Feb 2020 15:18:26 +0300 Subject: [PATCH 283/317] Use _throttle() in stopped state instead of sleep() --- freqtrade/worker.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index b3b3b712a..c397beaab 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -87,7 +87,7 @@ class Worker: logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: STOPPED.") self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: STOPPED.") - time.sleep(throttle_secs) + self._throttle(func=self._process_stopped, min_secs=throttle_secs) elif state == State.RUNNING: # Ping systemd watchdog before throttling @@ -95,7 +95,7 @@ class Worker: logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: RUNNING.") self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: RUNNING.") - self._throttle(func=self._process, min_secs=throttle_secs) + self._throttle(func=self._process_running, min_secs=throttle_secs) return state @@ -116,7 +116,11 @@ class Worker: time.sleep(duration) return result - def _process(self) -> None: + def _process_stopped(self) -> None: + # Maybe do here something in the future... + pass + + def _process_running(self) -> None: try: self.freqtrade.process() except TemporaryError as error: From 945ff09e27c31d5953231f6ecfa661434d6fb4fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Feb 2020 14:19:24 +0100 Subject: [PATCH 284/317] Use correct strategy path for docker testing --- build_helpers/publish_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 17d5230c9..013644563 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -23,7 +23,7 @@ if [ $? -ne 0 ]; then fi # Run backtest -docker run --rm -v $(pwd)/config.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy DefaultStrategy +docker run --rm -v $(pwd)/config.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy if [ $? -ne 0 ]; then echo "failed running backtest" From 04aa74e5add9425f73e2485a98a319c32b3ca2ad Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 03:37:38 +0300 Subject: [PATCH 285/317] Better throttling --- freqtrade/worker.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index c397beaab..40bfb54d8 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -108,12 +108,13 @@ class Worker: :return: Any """ start = time.time() - result = func(*args, **kwargs) logger.debug("========================================") - end = time.time() - duration = max(min_secs - (end - start), 0.0) - logger.debug(f"Throttling {func.__name__} for {duration:.2f} seconds") - time.sleep(duration) + result = func(*args, **kwargs) + time_passed = time.time() - start + sleep_duration = max(min_secs - time_passed, 0.0) + logger.debug(f"Throttling with '{func.__name__}()': sleep for {sleep_duration:.2f} s, " + f"last iteration took {time_passed:.2f} s.") + time.sleep(sleep_duration) return result def _process_stopped(self) -> None: From e0800b7c29fb0fe3b53afe2fa6732da7f37e06c1 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 03:52:14 +0300 Subject: [PATCH 286/317] Make throttle start time an worker object attribute --- freqtrade/worker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 40bfb54d8..dc8f9109f 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -32,6 +32,8 @@ class Worker: self._config = config self._init(False) + self.last_throttle_start_time: float = None + # Tell systemd that we completed initialization phase if self._sd_notify: logger.debug("sd_notify: READY=1") @@ -107,10 +109,10 @@ class Worker: :param min_secs: minimum execution time in seconds :return: Any """ - start = time.time() + self.last_throttle_start_time = time.time() logger.debug("========================================") result = func(*args, **kwargs) - time_passed = time.time() - start + time_passed = time.time() - self.last_throttle_start_time sleep_duration = max(min_secs - time_passed, 0.0) logger.debug(f"Throttling with '{func.__name__}()': sleep for {sleep_duration:.2f} s, " f"last iteration took {time_passed:.2f} s.") From 881f602f91b6afa5607f44eed8ad06165951ebe6 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 04:00:23 +0300 Subject: [PATCH 287/317] Adjust methods params --- freqtrade/worker.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index dc8f9109f..088526d85 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -32,7 +32,7 @@ class Worker: self._config = config self._init(False) - self.last_throttle_start_time: float = None + self.last_throttle_start_time: Optional[float] = None # Tell systemd that we completed initialization phase if self._sd_notify: @@ -65,15 +65,13 @@ class Worker: if state == State.RELOAD_CONF: self._reconfigure() - def _worker(self, old_state: Optional[State], throttle_secs: Optional[float] = None) -> State: + def _worker(self, old_state: Optional[State]) -> State: """ Trading routine that must be run at each loop :param old_state: the previous service state from the previous call :return: current service state """ state = self.freqtrade.state - if throttle_secs is None: - throttle_secs = self._throttle_secs # Log state transition if state != old_state: @@ -89,7 +87,7 @@ class Worker: logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: STOPPED.") self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: STOPPED.") - self._throttle(func=self._process_stopped, min_secs=throttle_secs) + self._throttle(func=self._process_stopped, throttle_secs=self._throttle_secs) elif state == State.RUNNING: # Ping systemd watchdog before throttling @@ -97,23 +95,23 @@ class Worker: logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: RUNNING.") self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: RUNNING.") - self._throttle(func=self._process_running, min_secs=throttle_secs) + self._throttle(func=self._process_running, throttle_secs=self._throttle_secs) return state - def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: + def _throttle(self, func: Callable[..., Any], throttle_secs: float, *args, **kwargs) -> Any: """ Throttles the given callable that it takes at least `min_secs` to finish execution. :param func: Any callable - :param min_secs: minimum execution time in seconds - :return: Any + :param throttle_secs: throttling interation execution time limit in seconds + :return: Any (result of execution of func) """ self.last_throttle_start_time = time.time() logger.debug("========================================") result = func(*args, **kwargs) time_passed = time.time() - self.last_throttle_start_time - sleep_duration = max(min_secs - time_passed, 0.0) + sleep_duration = max(throttle_secs - time_passed, 0.0) logger.debug(f"Throttling with '{func.__name__}()': sleep for {sleep_duration:.2f} s, " f"last iteration took {time_passed:.2f} s.") time.sleep(sleep_duration) From 269a669af82b586e69d061ed5866b8c56af173d0 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 05:07:31 +0300 Subject: [PATCH 288/317] Move heartbeat to worker --- freqtrade/freqtradebot.py | 10 ---------- freqtrade/worker.py | 16 ++++++++++++---- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 127586437..00d5c369a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -6,7 +6,6 @@ import logging import traceback from datetime import datetime from math import isclose -from os import getpid from threading import Lock from typing import Any, Dict, List, Optional, Tuple @@ -52,10 +51,6 @@ class FreqtradeBot: # Init objects self.config = config - self._heartbeat_msg = 0 - - self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) - self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) # Check config consistency here since strategies can set certain options @@ -159,11 +154,6 @@ class FreqtradeBot: self.check_handle_timedout() Trade.session.flush() - if (self.heartbeat_interval - and (arrow.utcnow().timestamp - self._heartbeat_msg > self.heartbeat_interval)): - logger.info(f"Bot heartbeat. PID={getpid()}") - self._heartbeat_msg = arrow.utcnow().timestamp - def _refresh_whitelist(self, trades: List[Trade] = []) -> List[str]: """ Refresh whitelist from pairlist or edge and extend it with trades. diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 088526d85..adce7ddda 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -4,8 +4,10 @@ Main Freqtrade worker class. import logging import time import traceback +from os import getpid from typing import Any, Callable, Dict, Optional +import arrow import sdnotify from freqtrade import __version__, constants @@ -33,6 +35,7 @@ class Worker: self._init(False) self.last_throttle_start_time: Optional[float] = None + self._heartbeat_msg = 0 # Tell systemd that we completed initialization phase if self._sd_notify: @@ -50,10 +53,10 @@ class Worker: # Init the instance of the bot self.freqtrade = FreqtradeBot(self._config) - self._throttle_secs = self._config.get('internals', {}).get( - 'process_throttle_secs', - constants.PROCESS_THROTTLE_SECS - ) + internals_config = self._config.get('internals', {}) + self._throttle_secs = internals_config.get('process_throttle_secs', + constants.PROCESS_THROTTLE_SECS) + self._heartbeat_interval = internals_config.get('heartbeat_interval', 60) self._sd_notify = sdnotify.SystemdNotifier() if \ self._config.get('internals', {}).get('sd_notify', False) else None @@ -97,6 +100,11 @@ class Worker: self._throttle(func=self._process_running, throttle_secs=self._throttle_secs) + if (self._heartbeat_interval + and (arrow.utcnow().timestamp - self._heartbeat_msg > self._heartbeat_interval)): + logger.info(f"Bot heartbeat. PID={getpid()}") + self._heartbeat_msg = arrow.utcnow().timestamp + return state def _throttle(self, func: Callable[..., Any], throttle_secs: float, *args, **kwargs) -> Any: From d2e20d86bb8808ac11427b08b9d2c3b1d27723bd Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 05:31:21 +0300 Subject: [PATCH 289/317] Align heartbeat to throttling logging --- freqtrade/worker.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index adce7ddda..523b9038f 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -7,7 +7,6 @@ import traceback from os import getpid from typing import Any, Callable, Dict, Optional -import arrow import sdnotify from freqtrade import __version__, constants @@ -34,8 +33,8 @@ class Worker: self._config = config self._init(False) - self.last_throttle_start_time: Optional[float] = None - self._heartbeat_msg = 0 + self.last_throttle_start_time: float = 0 + self._heartbeat_msg: float = 0 # Tell systemd that we completed initialization phase if self._sd_notify: @@ -100,10 +99,11 @@ class Worker: self._throttle(func=self._process_running, throttle_secs=self._throttle_secs) - if (self._heartbeat_interval - and (arrow.utcnow().timestamp - self._heartbeat_msg > self._heartbeat_interval)): - logger.info(f"Bot heartbeat. PID={getpid()}") - self._heartbeat_msg = arrow.utcnow().timestamp + if self._heartbeat_interval: + now = time.time() + if (now - self._heartbeat_msg) > self._heartbeat_interval: + logger.info(f"Bot heartbeat. PID={getpid()}") + self._heartbeat_msg = now return state From d9ecf3e4bfbdf2969c772ffb06e88ddecba4cf56 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 12:26:32 +0300 Subject: [PATCH 290/317] Add version and state to heartbeat message --- freqtrade/worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 523b9038f..f4b9f275b 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -102,7 +102,8 @@ class Worker: if self._heartbeat_interval: now = time.time() if (now - self._heartbeat_msg) > self._heartbeat_interval: - logger.info(f"Bot heartbeat. PID={getpid()}") + logger.info(f"Bot heartbeat. PID={getpid()}, " + f"version='{__version__}', state='{state.name}'") self._heartbeat_msg = now return state From f5b4a6d3d72fea870d912e6a1b28c47ccd3247dc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 11:03:25 +0100 Subject: [PATCH 291/317] Remove fetch_ticker caching --- freqtrade/exchange/exchange.py | 35 +++++++++++---------------------- freqtrade/freqtradebot.py | 4 ++-- tests/exchange/test_exchange.py | 15 +++----------- tests/rpc/test_rpc.py | 6 ------ 4 files changed, 16 insertions(+), 44 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b3b347016..e45238b07 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -66,8 +66,6 @@ class Exchange: self._config.update(config) - self._cached_ticker: Dict[str, Any] = {} - # Holds last candle refreshed time of each pair self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {} # Timestamp of last markets refresh @@ -591,28 +589,17 @@ class Exchange: raise OperationalException(e) from e @retrier - def fetch_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: - if refresh or pair not in self._cached_ticker.keys(): - try: - if pair not in self._api.markets or not self._api.markets[pair].get('active'): - raise DependencyException(f"Pair {pair} not available") - data = self._api.fetch_ticker(pair) - try: - self._cached_ticker[pair] = { - 'bid': float(data['bid']), - 'ask': float(data['ask']), - } - except KeyError: - logger.debug("Could not cache ticker data for %s", pair) - return data - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - else: - logger.info("returning cached ticker-data for %s", pair) - return self._cached_ticker[pair] + def fetch_ticker(self, pair: str) -> dict: + try: + if pair not in self._api.markets or not self._api.markets[pair].get('active'): + raise DependencyException(f"Pair {pair} not available") + data = self._api.fetch_ticker(pair) + return data + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 127586437..032e3e8f7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -253,7 +253,7 @@ class FreqtradeBot: else: if not tick: logger.info('Using Last Ask / Last Price') - ticker = self.exchange.fetch_ticker(pair, refresh) + ticker = self.exchange.fetch_ticker(pair) else: ticker = tick if ticker['ask'] < ticker['last']: @@ -631,7 +631,7 @@ class FreqtradeBot: rate = order_book['bids'][0][0] else: - rate = self.exchange.fetch_ticker(pair, refresh)['bid'] + rate = self.exchange.fetch_ticker(pair)['bid'] return rate def handle_trade(self, trade: Trade) -> bool: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8b2e439c3..3830d0acb 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1121,25 +1121,16 @@ def test_fetch_ticker(default_conf, mocker, exchange_name): assert ticker['bid'] == 0.5 assert ticker['ask'] == 1 - assert 'ETH/BTC' in exchange._cached_ticker - assert exchange._cached_ticker['ETH/BTC']['bid'] == 0.5 - assert exchange._cached_ticker['ETH/BTC']['ask'] == 1 - - # Test caching - api_mock.fetch_ticker = MagicMock() - exchange.fetch_ticker(pair='ETH/BTC', refresh=False) - assert api_mock.fetch_ticker.call_count == 0 - ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, "fetch_ticker", "fetch_ticker", - pair='ETH/BTC', refresh=True) + pair='ETH/BTC') api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.fetch_ticker(pair='ETH/BTC', refresh=True) + exchange.fetch_ticker(pair='ETH/BTC') with pytest.raises(DependencyException, match=r'Pair XRP/ETH not available'): - exchange.fetch_ticker(pair='XRP/ETH', refresh=True) + exchange.fetch_ticker(pair='XRP/ETH') @pytest.mark.parametrize("exchange_name", EXCHANGES) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index a35bfa0d6..2d1fa5b2d 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -67,8 +67,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) assert isnan(results[0]['current_rate']) @@ -136,8 +134,6 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] assert 'ETH/BTC' in result[0][1] @@ -262,8 +258,6 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Test non-available pair mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' From 97e6e5e9766c87bb728fe396f4f5119616eeda1a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 11:12:33 +0100 Subject: [PATCH 292/317] Implement caching in the correct place --- freqtrade/freqtradebot.py | 32 ++++++++++++++++++++++++++------ tests/rpc/test_rpc.py | 6 +++--- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 032e3e8f7..d9126370b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -11,6 +11,7 @@ from threading import Lock from typing import Any, Dict, List, Optional, Tuple import arrow +from cachetools import TTLCache from requests.exceptions import RequestException from freqtrade import __version__, constants, persistence @@ -54,6 +55,9 @@ class FreqtradeBot: self._heartbeat_msg = 0 + self._sell_rate_cache = TTLCache(maxsize=100, ttl=5) + self._buy_rate_cache = TTLCache(maxsize=100, ttl=5) + self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) @@ -234,11 +238,19 @@ class FreqtradeBot: return trades_created - def get_buy_rate(self, pair: str, refresh: bool, tick: Dict = None) -> float: + def get_buy_rate(self, pair: str, refresh: bool) -> float: """ Calculates bid target between current ask price and last price + :param pair: Pair to get rate for + :param refresh: allow cached data :return: float: Price """ + if not refresh: + rate = self._sell_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + return rate + config_bid_strategy = self.config.get('bid_strategy', {}) if 'use_order_book' in config_bid_strategy and\ config_bid_strategy.get('use_order_book', False): @@ -251,11 +263,8 @@ class FreqtradeBot: logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) used_rate = order_book_rate else: - if not tick: - logger.info('Using Last Ask / Last Price') - ticker = self.exchange.fetch_ticker(pair) - else: - ticker = tick + logger.info('Using Last Ask / Last Price') + ticker = self.exchange.fetch_ticker(pair) if ticker['ask'] < ticker['last']: ticker_rate = ticker['ask'] else: @@ -263,6 +272,8 @@ class FreqtradeBot: ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) used_rate = ticker_rate + self._buy_rate_cache[pair] = used_rate + return used_rate def get_trade_stake_amount(self, pair: str) -> float: @@ -621,8 +632,16 @@ class FreqtradeBot: The orderbook portion is only used for rpc messaging, which would otherwise fail for BitMex (has no bid/ask in fetch_ticker) or remain static in any other case since it's not updating. + :param pair: Pair to get rate for + :param refresh: allow cached data :return: Bid rate """ + if not refresh: + rate = self._sell_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + return rate + config_ask_strategy = self.config.get('ask_strategy', {}) if config_ask_strategy.get('use_order_book', False): logger.debug('Using order book to get sell rate') @@ -632,6 +651,7 @@ class FreqtradeBot: else: rate = self.exchange.fetch_ticker(pair)['bid'] + self._sell_rate_cache[pair] = rate return rate def handle_trade(self, trade: Trade) -> bool: diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 2d1fa5b2d..40b2d6627 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -65,7 +65,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_order': '(limit buy rem=0.00000000)' } == results[0] - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) @@ -132,7 +132,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert 'ETH/BTC' in result[0][1] assert '-0.59% (-0.09)' == result[0][3] - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] @@ -256,7 +256,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['best_rate'], 6.2) # Test non-available pair - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 From 77ef3240cd17b932f499a19e7820e39818cf1aa9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 11:16:20 +0100 Subject: [PATCH 293/317] Implement log messages --- freqtrade/freqtradebot.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d9126370b..4b54b79e1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -246,9 +246,10 @@ class FreqtradeBot: :return: float: Price """ if not refresh: - rate = self._sell_rate_cache.get(pair) + rate = self._buy_rate_cache.get(pair) # Check if cache has been invalidated if rate: + logger.info(f"Using cached buy rate for {pair}.") return rate config_bid_strategy = self.config.get('bid_strategy', {}) @@ -577,7 +578,7 @@ class FreqtradeBot: """ Sends rpc notification when a buy cancel occured. """ - current_rate = self.get_buy_rate(trade.pair, True) + current_rate = self.get_buy_rate(trade.pair, False) msg = { 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, @@ -640,6 +641,7 @@ class FreqtradeBot: rate = self._sell_rate_cache.get(pair) # Check if cache has been invalidated if rate: + logger.info(f"Using cached sell rate for {pair}.") return rate config_ask_strategy = self.config.get('ask_strategy', {}) @@ -1078,7 +1080,7 @@ class FreqtradeBot: """ profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) - current_rate = self.get_sell_rate(trade.pair, True) + current_rate = self.get_sell_rate(trade.pair, False) profit_percent = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_percent > 0 else "loss" From 2fe7b683cb3a11b4c629b7a796b30ff5838a5682 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 11:23:13 +0100 Subject: [PATCH 294/317] Add tests for cached rates --- tests/test_freqtradebot.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5ed4d296c..c7a70be8c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -915,13 +915,21 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: (5, 10, 1.0, 5), # last bigger than ask (5, 10, 0.5, 5), # last bigger than ask ]) -def test_get_buy_rate(mocker, default_conf, ask, last, last_ab, expected) -> None: +def test_get_buy_rate(mocker, default_conf, caplog, ask, last, last_ab, expected) -> None: default_conf['bid_strategy']['ask_last_balance'] = last_ab freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={'ask': ask, 'last': last})) assert freqtrade.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + assert freqtrade.get_buy_rate('ETH/BTC', False) == expected + assert log_has("Using cached buy rate for ETH/BTC.", caplog) + # Running a 2nd time with Refresh on! + caplog.clear() + assert freqtrade.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: @@ -3614,7 +3622,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order assert freqtrade.handle_trade(trade) is True -def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: +def test_get_sell_rate(default_conf, mocker, caplog, ticker, order_book_l2) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -3626,8 +3634,15 @@ def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: # Test regular mode ft = get_patched_freqtradebot(mocker, default_conf) rate = ft.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) assert rate == 0.00001098 + # Use caching + rate = ft.get_sell_rate(pair, False) + assert rate == 0.00001098 + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + caplog.clear() # Test orderbook mode default_conf['ask_strategy']['use_order_book'] = True @@ -3635,8 +3650,12 @@ def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: default_conf['ask_strategy']['order_book_max'] = 2 ft = get_patched_freqtradebot(mocker, default_conf) rate = ft.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) assert rate == 0.043936 + rate = ft.get_sell_rate(pair, False) + assert rate == 0.043936 + assert log_has("Using cached sell rate for ETH/BTC.", caplog) def test_startup_state(default_conf, mocker): From 7ecc56fa44d56d3ceaf70cc81b205cbcaff90795 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 13:10:41 +0100 Subject: [PATCH 295/317] Load ohlcv data as float --- freqtrade/data/history/jsondatahandler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 17b9fd7d7..7219d8c01 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -69,7 +69,7 @@ class JsonDataHandler(IDataHandler): filename = self._pair_data_filename(self._datadir, pair, timeframe) if not filename.exists(): return DataFrame(columns=self._columns) - pairdata = read_json(filename, orient='values') + pairdata = read_json(filename, orient='values', dtype='float64') pairdata.columns = self._columns pairdata['date'] = to_datetime(pairdata['date'], unit='ms', From 3186add87b82837018e8e9e46a0fb5c5d71a2b23 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 14:46:54 +0100 Subject: [PATCH 296/317] Use explicit column list for float parsing --- freqtrade/data/history/jsondatahandler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 7219d8c01..606018f34 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -69,7 +69,9 @@ class JsonDataHandler(IDataHandler): filename = self._pair_data_filename(self._datadir, pair, timeframe) if not filename.exists(): return DataFrame(columns=self._columns) - pairdata = read_json(filename, orient='values', dtype='float64') + pairdata = read_json(filename, orient='values', + dtype={'open': 'float', 'high': 'float', + 'low': 'float', 'close': 'float', 'volume': 'float'}) pairdata.columns = self._columns pairdata['date'] = to_datetime(pairdata['date'], unit='ms', From c651e0ac8278c854dc5901189d598b247e8d64ea Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 22 Feb 2020 19:46:40 +0300 Subject: [PATCH 297/317] Fix #2948 --- freqtrade/data/history/jsondatahandler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 606018f34..ee653d937 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -69,10 +69,11 @@ class JsonDataHandler(IDataHandler): filename = self._pair_data_filename(self._datadir, pair, timeframe) if not filename.exists(): return DataFrame(columns=self._columns) - pairdata = read_json(filename, orient='values', - dtype={'open': 'float', 'high': 'float', - 'low': 'float', 'close': 'float', 'volume': 'float'}) + pairdata = read_json(filename, orient='values') pairdata.columns = self._columns + pairdata = pairdata.astype(copy=False, + dtype={'open': 'float', 'high': 'float', + 'low': 'float', 'close': 'float', 'volume': 'float'}) pairdata['date'] = to_datetime(pairdata['date'], unit='ms', utc=True, From e2e6b940a3252985a7167ab49625e03131af0090 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 22 Feb 2020 19:54:19 +0300 Subject: [PATCH 298/317] copy=False does not make the changes inline anyway, so not needed --- freqtrade/data/history/jsondatahandler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index ee653d937..2b738a94a 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -71,8 +71,7 @@ class JsonDataHandler(IDataHandler): return DataFrame(columns=self._columns) pairdata = read_json(filename, orient='values') pairdata.columns = self._columns - pairdata = pairdata.astype(copy=False, - dtype={'open': 'float', 'high': 'float', + pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float', 'low': 'float', 'close': 'float', 'volume': 'float'}) pairdata['date'] = to_datetime(pairdata['date'], unit='ms', From ca8e52dc2cd61a80822cb158e1221d3d24ba62e7 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 23 Feb 2020 00:21:19 +0300 Subject: [PATCH 299/317] Show heartbeat message earlier after changing the state --- freqtrade/worker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index f4b9f275b..e17f61f2f 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -83,6 +83,10 @@ class Worker: if state == State.RUNNING: self.freqtrade.startup() + # Reset heartbeat timestamp to log the heartbeat message at + # first throttling iteration when the state changes + self._heartbeat_msg = 0 + if state == State.STOPPED: # Ping systemd watchdog before sleeping in the stopped state if self._sd_notify: From 259dc75a3083b76980bf12d6575764eac96dd202 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 22 Feb 2020 23:10:46 +0100 Subject: [PATCH 300/317] some order and added weighted BB indicator to list --- .../templates/subtemplates/indicators_full.j2 | 103 +++++++++++------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 879a2daa0..87b385dd0 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -2,12 +2,17 @@ # Momentum Indicators # ------------------------------------ -# RSI -dataframe['rsi'] = ta.RSI(dataframe) - # ADX dataframe['adx'] = ta.ADX(dataframe) +# # Plus Directional Indicator / Movement +# dataframe['plus_dm'] = ta.PLUS_DM(dataframe) +# dataframe['plus_di'] = ta.PLUS_DI(dataframe) + +# # Minus Directional Indicator / Movement +# dataframe['minus_dm'] = ta.MINUS_DM(dataframe) +# dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # # Aroon, Aroon Oscillator # aroon = ta.AROON(dataframe) # dataframe['aroonup'] = aroon['aroonup'] @@ -20,6 +25,31 @@ dataframe['adx'] = ta.ADX(dataframe) # # Commodity Channel Index: values Oversold:<-100, Overbought:>100 # dataframe['cci'] = ta.CCI(dataframe) +# RSI +dataframe['rsi'] = ta.RSI(dataframe) + +# # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy) +# rsi = 0.1 * (dataframe['rsi'] - 50) +# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) + +# # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy) +# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + +# # Stochastic Slow +# stoch = ta.STOCH(dataframe) +# dataframe['slowd'] = stoch['slowd'] +# dataframe['slowk'] = stoch['slowk'] + +# Stochastic Fast +stoch_fast = ta.STOCHF(dataframe) +dataframe['fastd'] = stoch_fast['fastd'] +dataframe['fastk'] = stoch_fast['fastk'] + +# # Stochastic RSI +# stoch_rsi = ta.STOCHRSI(dataframe) +# dataframe['fastd_rsi'] = stoch_rsi['fastd'] +# dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -29,60 +59,57 @@ dataframe['macdhist'] = macd['macdhist'] # MFI dataframe['mfi'] = ta.MFI(dataframe) -# # Minus Directional Indicator / Movement -# dataframe['minus_dm'] = ta.MINUS_DM(dataframe) -# dataframe['minus_di'] = ta.MINUS_DI(dataframe) - -# # Plus Directional Indicator / Movement -# dataframe['plus_dm'] = ta.PLUS_DM(dataframe) -# dataframe['plus_di'] = ta.PLUS_DI(dataframe) -# dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # # ROC # dataframe['roc'] = ta.ROC(dataframe) -# # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) -# rsi = 0.1 * (dataframe['rsi'] - 50) -# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) - -# # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) -# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - -# # Stoch -# stoch = ta.STOCH(dataframe) -# dataframe['slowd'] = stoch['slowd'] -# dataframe['slowk'] = stoch['slowk'] - -# Stoch fast -stoch_fast = ta.STOCHF(dataframe) -dataframe['fastd'] = stoch_fast['fastd'] -dataframe['fastk'] = stoch_fast['fastk'] - -# # Stoch RSI -# stoch_rsi = ta.STOCHRSI(dataframe) -# dataframe['fastd_rsi'] = stoch_rsi['fastd'] -# dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Overlap Studies # ------------------------------------ -# Bollinger bands +# Bollinger Bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] +dataframe["bb_percent"] = ( + (dataframe["close"] - dataframe["bb_lowerband"]) / + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) +) +dataframe["bb_width"] = ( + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"] +) + +# Bollinger Bands - Weighted (EMA based instead of SMA) +# weighted_bollinger = qtpylib.weighted_bollinger_bands( +# qtpylib.typical_price(dataframe), window=20, stds=2 +# ) +# dataframe["wbb_upperband"] = weighted_bollinger["upper"] +# dataframe["wbb_lowerband"] = weighted_bollinger["lower"] +# dataframe["wbb_middleband"] = weighted_bollinger["mid"] +# dataframe["wbb_percent"] = ( +# (dataframe["close"] - dataframe["wbb_lowerband"]) / +# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) +# ) +# dataframe["wbb_width"] = ( +# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / dataframe["wbb_middleband"] +# ) # # EMA - Exponential Moving Average # dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) # dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) # dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) +# dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21) # dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) # dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) # # SMA - Simple Moving Average -# dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) +# dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3) +# dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5) +# dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10) +# dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21) +# dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50) +# dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) -# SAR Parabol +# Parabolic SAR dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average @@ -142,7 +169,7 @@ dataframe['htleadsine'] = hilbert['leadsine'] # # Chart type # # ------------------------------------ -# # Heikinashi stategy +# # Heikin Ashi Strategy # heikinashi = qtpylib.heikinashi(dataframe) # dataframe['ha_open'] = heikinashi['open'] # dataframe['ha_close'] = heikinashi['close'] From b49b9b515ed14d1c501e9350fe727f0f16a7722d Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 22 Feb 2020 23:37:15 +0100 Subject: [PATCH 301/317] final touches --- freqtrade/templates/subtemplates/indicators_full.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 87b385dd0..cd106451e 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -19,7 +19,7 @@ dataframe['adx'] = ta.ADX(dataframe) # dataframe['aroondown'] = aroon['aroondown'] # dataframe['aroonosc'] = ta.AROONOSC(dataframe) -# # Awesome oscillator +# # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) # # Commodity Channel Index: values Oversold:<-100, Overbought:>100 From 2957756275367bcd35734989cfe61558356a4a09 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 22 Feb 2020 23:39:01 +0100 Subject: [PATCH 302/317] final touches plus --- freqtrade/templates/subtemplates/indicators_full.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index cd106451e..903aebb73 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -22,7 +22,7 @@ dataframe['adx'] = ta.ADX(dataframe) # # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) -# # Commodity Channel Index: values Oversold:<-100, Overbought:>100 +# # Commodity Channel Index: values Oversold:-100, Overbought:100 # dataframe['cci'] = ta.CCI(dataframe) # RSI From 5ac624446587f95af22b322b53caddd87692f021 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 22 Feb 2020 23:50:26 +0100 Subject: [PATCH 303/317] added keltner channel and uo --- .../templates/subtemplates/indicators_full.j2 | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 903aebb73..af472faef 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -22,7 +22,23 @@ dataframe['adx'] = ta.ADX(dataframe) # # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) -# # Commodity Channel Index: values Oversold:-100, Overbought:100 +# # Keltner Channel +# keltner = qtpylib.keltner_channel(dataframe) +# dataframe["kc_upperband"] = keltner["upper"] +# dataframe["kc_lowerband"] = keltner["lower"] +# dataframe["kc_middleband"] = keltner["mid"] +# dataframe["kc_percent"] = ( +# (dataframe["close"] - dataframe["kc_lowerband"]) / +# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) +# ) +# dataframe["kc_width"] = ( +# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"] +# ) + +# # Ultimate Oscillator +# dataframe['ao'] = ta.ULTOSC(dataframe) + +# # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) # RSI From d2181bdd9492d16be409d82cd2fa5f0782c15838 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 23 Feb 2020 01:45:15 +0300 Subject: [PATCH 304/317] Adjust tests --- tests/test_freqtradebot.py | 28 ++--------------- tests/test_worker.py | 62 ++++++++++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 35 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5ed4d296c..20db46fac 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -782,7 +782,7 @@ def test_process_exchange_failures(default_conf, ticker, mocker) -> None: worker = Worker(args=None, config=default_conf) patch_get_signal(worker.freqtrade) - worker._process() + worker._process_running() assert sleep_mock.has_calls() @@ -799,7 +799,7 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None: assert worker.freqtrade.state == State.RUNNING - worker._process() + worker._process_running() assert worker.freqtrade.state == State.STOPPED assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] @@ -3665,30 +3665,6 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): assert reinit_mock.call_count == 0 -def test_process_i_am_alive(default_conf, mocker, caplog): - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - - ftbot = get_patched_freqtradebot(mocker, default_conf) - message = r"Bot heartbeat\. PID=.*" - ftbot.process() - assert log_has_re(message, caplog) - assert ftbot._heartbeat_msg != 0 - - caplog.clear() - # Message is not shown before interval is up - ftbot.process() - assert not log_has_re(message, caplog) - - caplog.clear() - # Set clock - 70 seconds - ftbot._heartbeat_msg -= 70 - - ftbot.process() - assert log_has_re(message, caplog) - - @pytest.mark.usefixtures("init_persistence") def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, caplog): default_conf['dry_run'] = True diff --git a/tests/test_worker.py b/tests/test_worker.py index 2fb42d47e..7b446ac6a 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, PropertyMock from freqtrade.data.dataprovider import DataProvider from freqtrade.state import State from freqtrade.worker import Worker -from tests.conftest import get_patched_worker, log_has +from tests.conftest import get_patched_worker, log_has, log_has_re def test_worker_state(mocker, default_conf, markets) -> None: @@ -38,15 +38,13 @@ def test_worker_running(mocker, default_conf, caplog) -> None: def test_worker_stopped(mocker, default_conf, caplog) -> None: mock_throttle = MagicMock() mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) - mock_sleep = mocker.patch('time.sleep', return_value=None) worker = get_patched_worker(mocker, default_conf) worker.freqtrade.state = State.STOPPED state = worker._worker(old_state=State.RUNNING) assert state is State.STOPPED assert log_has('Changing state to: STOPPED', caplog) - assert mock_throttle.call_count == 0 - assert mock_sleep.call_count == 1 + assert mock_throttle.call_count == 1 def test_throttle(mocker, default_conf, caplog) -> None: @@ -57,14 +55,14 @@ def test_throttle(mocker, default_conf, caplog) -> None: worker = get_patched_worker(mocker, default_conf) start = time.time() - result = worker._throttle(throttled_func, min_secs=0.1) + result = worker._throttle(throttled_func, throttle_secs=0.1) end = time.time() assert result == 42 assert end - start > 0.1 - assert log_has('Throttling throttled_func for 0.10 seconds', caplog) + assert log_has_re(r"Throttling with 'throttled_func\(\)': sleep for 0\.10 s.*", caplog) - result = worker._throttle(throttled_func, min_secs=-1) + result = worker._throttle(throttled_func, throttle_secs=-1) assert result == 42 @@ -74,8 +72,54 @@ def test_throttle_with_assets(mocker, default_conf) -> None: worker = get_patched_worker(mocker, default_conf) - result = worker._throttle(throttled_func, min_secs=0.1, nb_assets=666) + result = worker._throttle(throttled_func, throttle_secs=0.1, nb_assets=666) assert result == 666 - result = worker._throttle(throttled_func, min_secs=0.1) + result = worker._throttle(throttled_func, throttle_secs=0.1) assert result == -1 + + +def test_worker_heartbeat_running(default_conf, mocker, caplog): + message = r"Bot heartbeat\. PID=.*state='RUNNING'" + + mock_throttle = MagicMock() + mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + worker = get_patched_worker(mocker, default_conf) + + worker.freqtrade.state = State.RUNNING + worker._worker(old_state=State.STOPPED) + assert log_has_re(message, caplog) + + caplog.clear() + # Message is not shown before interval is up + worker._worker(old_state=State.RUNNING) + assert not log_has_re(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + worker._heartbeat_msg -= 70 + worker._worker(old_state=State.RUNNING) + assert log_has_re(message, caplog) + + +def test_worker_heartbeat_stopped(default_conf, mocker, caplog): + message = r"Bot heartbeat\. PID=.*state='STOPPED'" + + mock_throttle = MagicMock() + mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + worker = get_patched_worker(mocker, default_conf) + + worker.freqtrade.state = State.STOPPED + worker._worker(old_state=State.RUNNING) + assert log_has_re(message, caplog) + + caplog.clear() + # Message is not shown before interval is up + worker._worker(old_state=State.STOPPED) + assert not log_has_re(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + worker._heartbeat_msg -= 70 + worker._worker(old_state=State.STOPPED) + assert log_has_re(message, caplog) From e04c2dda2cfd3c18690be04dd3b22f98731b0b04 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 22 Feb 2020 23:58:31 +0100 Subject: [PATCH 305/317] fixed typo --- freqtrade/templates/subtemplates/indicators_full.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index af472faef..60a358bec 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -36,7 +36,7 @@ dataframe['adx'] = ta.ADX(dataframe) # ) # # Ultimate Oscillator -# dataframe['ao'] = ta.ULTOSC(dataframe) +# dataframe['uo'] = ta.ULTOSC(dataframe) # # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) From f25d6224ddeab8a3889daa69c1fac8eb375d169b Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sun, 23 Feb 2020 16:22:19 +0100 Subject: [PATCH 306/317] modified sample_strategy --- freqtrade/templates/sample_strategy.py | 136 ++++++++++++++++--------- 1 file changed, 90 insertions(+), 46 deletions(-) diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index 92f6aefba..8a4b27c72 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -124,11 +124,16 @@ class SampleStrategy(IStrategy): # Momentum Indicators # ------------------------------------ - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - # ADX - dataframe['adx'] = ta.ADX(dataframe) + # dataframe['adx'] = ta.ADX(dataframe) + + # # Plus Directional Indicator / Movement + # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + # dataframe['plus_di'] = ta.PLUS_DI(dataframe) + + # # Minus Directional Indicator / Movement + # dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + # dataframe['minus_di'] = ta.MINUS_DI(dataframe) # # Aroon, Aroon Oscillator # aroon = ta.AROON(dataframe) @@ -136,12 +141,53 @@ class SampleStrategy(IStrategy): # dataframe['aroondown'] = aroon['aroondown'] # dataframe['aroonosc'] = ta.AROONOSC(dataframe) - # # Awesome oscillator + # # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - # # Commodity Channel Index: values Oversold:<-100, Overbought:>100 + # # Keltner Channel + # keltner = qtpylib.keltner_channel(dataframe) + # dataframe["kc_upperband"] = keltner["upper"] + # dataframe["kc_lowerband"] = keltner["lower"] + # dataframe["kc_middleband"] = keltner["mid"] + # dataframe["kc_percent"] = ( + # (dataframe["close"] - dataframe["kc_lowerband"]) / + # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) + # ) + # dataframe["kc_width"] = ( + # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"] + # ) + + # # Ultimate Oscillator + # dataframe['uo'] = ta.ULTOSC(dataframe) + + # # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy) + # rsi = 0.1 * (dataframe['rsi'] - 50) + # dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) + + # # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy) + # dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + + # # Stochastic Slow + # stoch = ta.STOCH(dataframe) + # dataframe['slowd'] = stoch['slowd'] + # dataframe['slowk'] = stoch['slowk'] + + # Stochastic Fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # # Stochastic RSI + # stoch_rsi = ta.STOCHRSI(dataframe) + # dataframe['fastd_rsi'] = stoch_rsi['fastd'] + # dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -151,71 +197,69 @@ class SampleStrategy(IStrategy): # MFI dataframe['mfi'] = ta.MFI(dataframe) - # # Minus Directional Indicator / Movement - # dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - # dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # # Plus Directional Indicator / Movement - # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - # dataframe['plus_di'] = ta.PLUS_DI(dataframe) - # dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # # ROC # dataframe['roc'] = ta.ROC(dataframe) - # # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - # rsi = 0.1 * (dataframe['rsi'] - 50) - # dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) - - # # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - # dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - - # # Stoch - # stoch = ta.STOCH(dataframe) - # dataframe['slowd'] = stoch['slowd'] - # dataframe['slowk'] = stoch['slowk'] - - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - - # # Stoch RSI - # stoch_rsi = ta.STOCHRSI(dataframe) - # dataframe['fastd_rsi'] = stoch_rsi['fastd'] - # dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Overlap Studies # ------------------------------------ - # Bollinger bands + # Bollinger Bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] + dataframe["bb_percent"] = ( + (dataframe["close"] - dataframe["bb_lowerband"]) / + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) + ) + dataframe["bb_width"] = ( + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"] + ) + + # Bollinger Bands - Weighted (EMA based instead of SMA) + # weighted_bollinger = qtpylib.weighted_bollinger_bands( + # qtpylib.typical_price(dataframe), window=20, stds=2 + # ) + # dataframe["wbb_upperband"] = weighted_bollinger["upper"] + # dataframe["wbb_lowerband"] = weighted_bollinger["lower"] + # dataframe["wbb_middleband"] = weighted_bollinger["mid"] + # dataframe["wbb_percent"] = ( + # (dataframe["close"] - dataframe["wbb_lowerband"]) / + # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) + # ) + # dataframe["wbb_width"] = ( + # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / + # dataframe["wbb_middleband"] + # ) # # EMA - Exponential Moving Average # dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) # dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) # dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + # dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21) # dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) # dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) # # SMA - Simple Moving Average - # dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3) + # dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5) + # dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10) + # dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21) + # dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50) + # dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) - # SAR Parabol - dataframe['sar'] = ta.SAR(dataframe) + # Parabolic SAR + # dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) # Cycle Indicator # ------------------------------------ # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] + # hilbert = ta.HT_SINE(dataframe) + # dataframe['htsine'] = hilbert['sine'] + # dataframe['htleadsine'] = hilbert['leadsine'] # Pattern Recognition - Bullish candlestick patterns # ------------------------------------ @@ -264,7 +308,7 @@ class SampleStrategy(IStrategy): # # Chart type # # ------------------------------------ - # # Heikinashi stategy + # # Heikin Ashi Strategy # heikinashi = qtpylib.heikinashi(dataframe) # dataframe['ha_open'] = heikinashi['open'] # dataframe['ha_close'] = heikinashi['close'] From 0eeafcd157c17d6f7de92ab66c6f267c454bcec2 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sun, 23 Feb 2020 16:56:55 +0100 Subject: [PATCH 307/317] matched commenting on previous sample_strategy.py --- freqtrade/templates/sample_strategy.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index 8a4b27c72..17372e1e0 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -125,7 +125,7 @@ class SampleStrategy(IStrategy): # ------------------------------------ # ADX - # dataframe['adx'] = ta.ADX(dataframe) + dataframe['adx'] = ta.ADX(dataframe) # # Plus Directional Indicator / Movement # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) @@ -249,17 +249,17 @@ class SampleStrategy(IStrategy): # dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) # Parabolic SAR - # dataframe['sar'] = ta.SAR(dataframe) + dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average - # dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) # Cycle Indicator # ------------------------------------ # Hilbert Transform Indicator - SineWave - # hilbert = ta.HT_SINE(dataframe) - # dataframe['htsine'] = hilbert['sine'] - # dataframe['htleadsine'] = hilbert['leadsine'] + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] # Pattern Recognition - Bullish candlestick patterns # ------------------------------------ From e545ef563c0b5aaafc618bc3501c1d1e43f68c07 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sun, 23 Feb 2020 22:50:58 +0300 Subject: [PATCH 308/317] Wording adjusted in helpstring --- freqtrade/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index e17f61f2f..4c28ecaeb 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -69,7 +69,7 @@ class Worker: def _worker(self, old_state: Optional[State]) -> State: """ - Trading routine that must be run at each loop + The main routine that runs each throttling iteration and handles the states. :param old_state: the previous service state from the previous call :return: current service state """ From 353f722dc54dcd6eab646a844596383eccc5d326 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 08:04:07 +0000 Subject: [PATCH 309/317] Bump requests from 2.22.0 to 2.23.0 Bumps [requests](https://github.com/psf/requests) from 2.22.0 to 2.23.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.22.0...v2.23.0) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 2be51ba73..61809c698 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -5,7 +5,7 @@ SQLAlchemy==1.3.13 python-telegram-bot==12.4.2 arrow==0.15.5 cachetools==4.0.0 -requests==2.22.0 +requests==2.23.0 urllib3==1.25.8 wrapt==1.12.0 jsonschema==3.2.0 From 4054dec7a029594503222de5af6677669956d1cd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 08:04:32 +0000 Subject: [PATCH 310/317] Bump plotly from 4.5.0 to 4.5.1 Bumps [plotly](https://github.com/plotly/plotly.py) from 4.5.0 to 4.5.1. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v4.5.0...v4.5.1) Signed-off-by: dependabot-preview[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 26467d90b..5e62a5e95 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.5.0 +plotly==4.5.1 From ff69b511e311bb73d36d0d2661f7e4100a46d283 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 08:04:44 +0000 Subject: [PATCH 311/317] Bump scikit-optimize from 0.7.2 to 0.7.4 Bumps [scikit-optimize](https://github.com/scikit-optimize/scikit-optimize) from 0.7.2 to 0.7.4. - [Release notes](https://github.com/scikit-optimize/scikit-optimize/releases) - [Changelog](https://github.com/scikit-optimize/scikit-optimize/blob/master/CHANGELOG.md) - [Commits](https://github.com/scikit-optimize/scikit-optimize/compare/v0.7.2...v0.7.4) Signed-off-by: dependabot-preview[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index e97e7f6be..2984229c1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -4,6 +4,6 @@ # Required for hyperopt scipy==1.4.1 scikit-learn==0.22.1 -scikit-optimize==0.7.2 +scikit-optimize==0.7.4 filelock==3.0.12 joblib==0.14.1 From d63aaf3bfd7eb511387625b76435ea873e51e444 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 08:05:15 +0000 Subject: [PATCH 312/317] Bump ccxt from 1.22.61 to 1.22.95 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.22.61 to 1.22.95. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.22.61...1.22.95) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 2be51ba73..f792f5348 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.22.61 +ccxt==1.22.95 SQLAlchemy==1.3.13 python-telegram-bot==12.4.2 arrow==0.15.5 From 23b47b66eccb408d56ba0abd6e6e5ab71b8ae3f5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 Feb 2020 20:11:25 +0100 Subject: [PATCH 313/317] Update install-script documentation and reorder installation steps --- docs/installation.md | 8 ++++---- mkdocs.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 054cafe9b..0feaf509d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -65,11 +65,11 @@ usage: ** --install ** -With this option, the script will install everything you need to run the bot: +With this option, the script will install the bot and most dependencies: +You will need to have git and python3.6+ installed beforehand for this to work. * Mandatory software as: `ta-lib` -* Setup your virtualenv -* Configure your `config.json` file +* Setup your virtualenv under `.env/` This option is a combination of installation tasks, `--reset` and `--config`. @@ -83,7 +83,7 @@ This option will hard reset your branch (only if you are on either `master` or ` ** --config ** -Use this option to configure the `config.json` configuration file. The script will interactively ask you questions to setup your bot and create your `config.json`. +DEPRECATED - use `freqtrade new-config -c config.json` instead. ------ diff --git a/mkdocs.yml b/mkdocs.yml index d53687c64..4e7e6ff75 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,8 +1,8 @@ site_name: Freqtrade nav: - - About: index.md - - Installation: installation.md + - Home: index.md - Installation Docker: docker.md + - Installation: installation.md - Configuration: configuration.md - Strategy Customization: strategy-customization.md - Stoploss: stoploss.md From 2f349e0504d014e01f428e8fc0b6e8f8a619aeef Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 Feb 2020 20:21:25 +0100 Subject: [PATCH 314/317] Improve install documentation by streamlining the process --- docs/configuration.md | 14 +++++++++++--- docs/installation.md | 12 +++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0b9519688..234ff49ba 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -626,6 +626,11 @@ In production mode, the bot will engage your money. Be careful, since a wrong strategy can lose all your money. Be aware of what you are doing when you run it in production mode. +### Setup your exchange account + +You will need to create API Keys (usually you get `key` and `secret`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. +API Keys are usually only required for real / production trading, but are not required for paper-trading / dry-run. + ### To switch your bot in production mode **Edit your `config.json` file.** @@ -647,11 +652,14 @@ you run it in production mode. } ``` -!!! Note - If you have an exchange API key yet, [see our tutorial](installation.md#setup-your-exchange-account). - You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange. +### Setup your exchange account + +You will need to create API Keys (usually you get `key` and `secret`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. +API Keys are usually only required for real / production trading, but are not required for paper-trading / dry-run. + + ### Using proxy with Freqtrade To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. diff --git a/docs/installation.md b/docs/installation.md index 0feaf509d..5a15be234 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,6 +2,8 @@ This page explains how to prepare your environment for running the bot. +Please consider using the prebuilt [docker images](docker.md) to get started quickly while trying out freqtrade. + ## Prerequisite ### Requirements @@ -14,15 +16,7 @@ Click each one for install guide: * [virtualenv](https://virtualenv.pypa.io/en/stable/installation/) (Recommended) * [TA-Lib](https://mrjbq7.github.io/ta-lib/install.html) (install instructions below) -### API keys - -Before running your bot in production you will need to setup few -external API. In production mode, the bot will require valid Exchange API -credentials. We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot) (optional but recommended). - -### Setup your exchange account - -You will need to create API Keys (Usually you get `key` and `secret`) from the Exchange website and insert this into the appropriate fields in the configuration or when asked by the installation script. + We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot), which is optional but recommended. ## Quick start From a29653b510722251debc426981e3afc1d9929a4d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Feb 2020 08:59:27 +0100 Subject: [PATCH 315/317] Wording changes to install docs Co-Authored-By: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> --- docs/configuration.md | 4 ++-- docs/installation.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 234ff49ba..3844f2812 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -656,8 +656,8 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d ### Setup your exchange account -You will need to create API Keys (usually you get `key` and `secret`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. -API Keys are usually only required for real / production trading, but are not required for paper-trading / dry-run. +You will need to create API Keys (usually you get `key` and `secret`, some exchanges supply it with `password`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. +API Keys are usually only required for live trade (trading for real money, bot running in the "production mode", executing real orders on the exchange) and are not required for the bot running in the dry-run (trade simulation) mode. When you setup the bot in the dry-run mode, you may fill these fields with empty values. ### Using proxy with Freqtrade diff --git a/docs/installation.md b/docs/installation.md index 5a15be234..88e2ef6eb 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,7 +2,7 @@ This page explains how to prepare your environment for running the bot. -Please consider using the prebuilt [docker images](docker.md) to get started quickly while trying out freqtrade. +Please consider using the prebuilt [docker images](docker.md) to get started quickly while trying out freqtrade evaluating how it operates. ## Prerequisite From 8ae0f99a960fa4e7bdb615fa19c29fac887cb9e8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Feb 2020 09:05:48 +0100 Subject: [PATCH 316/317] Remove duplicate section --- docs/configuration.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3844f2812..e0dc43f5d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -628,8 +628,8 @@ you run it in production mode. ### Setup your exchange account -You will need to create API Keys (usually you get `key` and `secret`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. -API Keys are usually only required for real / production trading, but are not required for paper-trading / dry-run. +You will need to create API Keys (usually you get `key` and `secret`, some exchanges require an additional `password`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the `freqtrade new-config` command. +API Keys are usually only required for live trading (trading for real money, bot running in "production mode", executing real orders on the exchange) and are not required for the bot running in dry-run (trade simulation) mode. When you setup the bot in dry-run mode, you may fill these fields with empty values. ### To switch your bot in production mode @@ -654,12 +654,6 @@ API Keys are usually only required for real / production trading, but are not re You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange. -### Setup your exchange account - -You will need to create API Keys (usually you get `key` and `secret`, some exchanges supply it with `password`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. -API Keys are usually only required for live trade (trading for real money, bot running in the "production mode", executing real orders on the exchange) and are not required for the bot running in the dry-run (trade simulation) mode. When you setup the bot in the dry-run mode, you may fill these fields with empty values. - - ### Using proxy with Freqtrade To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. From a6b48f7366c749693fce9144266bb6a74ffdc2d5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Feb 2020 15:16:55 +0100 Subject: [PATCH 317/317] Version bump 2020.02 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 1a7cb087e..bb1321237 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2020.01' +__version__ = '2020.02' if __version__ == 'develop':