From b7553d20d41f46dd9eba335330c649ceb83b1fb7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 19 Aug 2022 13:44:31 +0200 Subject: [PATCH 1/2] Get min/max data in list-data command --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 5 ++++ freqtrade/commands/data_commands.py | 37 +++++++++++++++++------- freqtrade/configuration/configuration.py | 3 ++ freqtrade/data/history/idatahandler.py | 14 ++++++++- tests/commands/test_commands.py | 21 ++++++++++++++ 6 files changed, 69 insertions(+), 13 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 48a423be4..266095cfa 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -69,7 +69,7 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "exchange", "tradin ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] -ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"] +ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive", "timerange", "download_trades", "exchange", "timeframes", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index f85b75af1..706f28f02 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -434,6 +434,11 @@ AVAILABLE_CLI_OPTIONS = { help='Storage format for downloaded trades data. (default: `jsongz`).', choices=constants.AVAILABLE_DATAHANDLERS, ), + "show_timerange": Arg( + '--show-timerange', + help='Show timerange available for available data. (May take a while to calculate).', + action='store_true', + ), "exchange": Arg( '--exchange', help=f'Exchange name (default: `{constants.DEFAULT_EXCHANGE}`). ' diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 311590e64..36a86bece 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta from typing import Any, Dict, List from freqtrade.configuration import TimeRange, setup_utils_configuration +from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) @@ -177,17 +178,31 @@ def start_list_data(args: Dict[str, Any]) -> None: paircombs = [comb for comb in paircombs if comb[0] in args['pairs']] print(f"Found {len(paircombs)} pair / timeframe combinations.") - groupedpair = defaultdict(list) - for pair, timeframe, candle_type in sorted( - paircombs, - key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]) - ): - groupedpair[(pair, candle_type)].append(timeframe) + if not config.get('show_timerange'): + groupedpair = defaultdict(list) + for pair, timeframe, candle_type in sorted( + paircombs, + key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]) + ): + groupedpair[(pair, candle_type)].append(timeframe) - if groupedpair: + if groupedpair: + print(tabulate([ + (pair, ', '.join(timeframes), candle_type) + for (pair, candle_type), timeframes in groupedpair.items() + ], + headers=("Pair", "Timeframe", "Type"), + tablefmt='psql', stralign='right')) + else: + paircombs1 = [( + pair, timeframe, candle_type, + *dhc.ohlcv_data_min_max(pair, timeframe, candle_type) + ) for pair, timeframe, candle_type in paircombs] print(tabulate([ - (pair, ', '.join(timeframes), candle_type) - for (pair, candle_type), timeframes in groupedpair.items() - ], - headers=("Pair", "Timeframe", "Type"), + (pair, timeframe, candle_type, + start.strftime(DATETIME_PRINT_FORMAT), + end.strftime(DATETIME_PRINT_FORMAT)) + for pair, timeframe, candle_type, start, end in paircombs1 + ], + headers=("Pair", "Timeframe", "Type", 'From', 'To'), tablefmt='psql', stralign='right')) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index b4f36aa3c..0db585acc 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -426,6 +426,9 @@ class Configuration: self._args_to_config(config, argname='dataformat_trades', logstring='Using "{}" to store trades data.') + self._args_to_config(config, argname='show_timerange', + logstring='Detected --show-timerange') + def _process_data_options(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='new_pairs_days', logstring='Detected --new-pairs-days: {}') diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index eb2441abe..846bcc607 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -9,7 +9,7 @@ from abc import ABC, abstractmethod from copy import deepcopy from datetime import datetime, timezone from pathlib import Path -from typing import List, Optional, Type +from typing import List, Optional, Tuple, Type from pandas import DataFrame @@ -84,6 +84,18 @@ class IDataHandler(ABC): :return: None """ + def ohlcv_data_min_max(self, pair: str, timeframe: str, + candle_type: CandleType) -> Tuple[datetime, datetime]: + """ + Returns the min and max timestamp for the given pair and timeframe. + :param pair: Pair to get min/max for + :param timeframe: Timeframe to get min/max for + :param candle_type: Any of the enum CandleType (must match trading mode!) + :return: (min, max) + """ + data = self._ohlcv_load(pair, timeframe, None, candle_type) + return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime() + @abstractmethod def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange], candle_type: CandleType diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 9df6acf75..28515a28a 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1430,6 +1430,27 @@ def test_start_list_data(testdatadir, capsys): assert "\n| XRP/USDT | 1h | futures |\n" in captured.out assert "\n| XRP/USDT | 1h, 8h | mark |\n" in captured.out + args = [ + "list-data", + "--data-format-ohlcv", + "json", + "--pairs", "XRP/ETH", + "--datadir", + str(testdatadir), + "--show-timerange", + ] + pargs = get_args(args) + pargs['config'] = None + start_list_data(pargs) + captured = capsys.readouterr() + assert "Found 2 pair / timeframe combinations." in captured.out + assert ("\n| Pair | Timeframe | Type | From | To |\n" + in captured.out) + assert "UNITTEST/BTC" not in captured.out + assert ( + "\n| XRP/ETH | 1m | spot | 2019-10-11 00:00:00 | 2019-10-13 11:19:00 |\n" + in captured.out) + @pytest.mark.usefixtures("init_persistence") def test_show_trades(mocker, fee, capsys, caplog): From 1389c8f5b617f767b566b579dc8048be37cde97b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 19 Aug 2022 15:34:10 +0200 Subject: [PATCH 2/2] Update documentation with show-timerange option --- docs/data-download.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/data-download.md b/docs/data-download.md index 681fb717d..55c2ad738 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -374,6 +374,7 @@ usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--data-format-ohlcv {json,jsongz,hdf5}] [-p PAIRS [PAIRS ...]] [--trading-mode {spot,margin,futures}] + [--show-timerange] optional arguments: -h, --help show this help message and exit @@ -387,6 +388,8 @@ optional arguments: separated. --trading-mode {spot,margin,futures} Select Trading mode + --show-timerange Show timerange available for available data. (May take + a while to calculate). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages).