freqtrade_origin/freqtrade/commands/data_commands.py

218 lines
8.9 KiB
Python
Raw Normal View History

2020-01-26 12:17:26 +00:00
import logging
import sys
2020-07-12 07:57:00 +00:00
from collections import defaultdict
from datetime import datetime, timedelta
2020-01-26 12:17:26 +00:00
from typing import Any, Dict, List
from freqtrade.configuration import TimeRange, setup_utils_configuration
2023-02-25 19:50:26 +00:00
from freqtrade.constants import DATETIME_PRINT_FORMAT, Config
2020-09-28 17:39:41 +00:00
from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
2020-01-26 12:17:26 +00:00
refresh_backtest_trades_data)
2022-03-03 06:06:13 +00:00
from freqtrade.enums import CandleType, RunMode, TradingMode
2020-01-26 12:17:26 +00:00
from freqtrade.exceptions import OperationalException
2022-08-25 05:08:58 +00:00
from freqtrade.exchange import market_is_active, timeframe_to_minutes
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
2020-01-26 12:17:26 +00:00
from freqtrade.resolvers import ExchangeResolver
from freqtrade.util.binance_mig import migrate_binance_futures_data
2020-01-26 12:17:26 +00:00
2020-09-28 17:39:41 +00:00
2020-01-26 12:17:26 +00:00
logger = logging.getLogger(__name__)
2023-02-25 19:50:26 +00:00
def _data_download_sanity(config: Config) -> None:
if 'days' in config and 'timerange' in config:
raise OperationalException("--days and --timerange are mutually exclusive. "
"You can only specify one or the other.")
if 'pairs' not in config:
raise OperationalException(
"Downloading data requires a list of pairs. "
"Please check the documentation on how to configure this.")
2020-01-26 12:17:26 +00:00
def start_download_data(args: Dict[str, Any]) -> None:
"""
Download data (former download_backtest_data.py script)
"""
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
2023-02-25 19:50:26 +00:00
_data_download_sanity(config)
2020-01-26 12:17:26 +00:00
timerange = TimeRange()
if 'days' in config:
time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d")
2020-01-26 12:17:26 +00:00
timerange = TimeRange.parse_timerange(f'{time_since}-')
if 'timerange' in config:
timerange = timerange.parse_timerange(config['timerange'])
# Remove stake-currency to skip checks which are not relevant for datadownload
config['stake_currency'] = ''
2020-01-26 12:17:26 +00:00
pairs_not_available: List[str] = []
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
markets = [p for p, m in exchange.markets.items() if market_is_active(m)
or config.get('include_inactive')]
expanded_pairs = dynamic_expand_pairlist(config, markets)
# Manual validations of relevant settings
if not config['exchange'].get('skip_pair_validation', False):
exchange.validate_pairs(expanded_pairs)
2021-01-12 00:13:58 +00:00
logger.info(f"About to download pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
for timeframe in config['timeframes']:
exchange.validate_timeframes(timeframe)
2020-01-26 12:17:26 +00:00
try:
if config.get('download_trades'):
if config.get('trading_mode') == 'futures':
raise OperationalException("Trade download not supported for futures.")
2020-01-26 12:17:26 +00:00
pairs_not_available = refresh_backtest_trades_data(
2021-01-12 00:13:58 +00:00
exchange, pairs=expanded_pairs, datadir=config['datadir'],
timerange=timerange, new_pairs_days=config['new_pairs_days'],
erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
2020-01-26 12:17:26 +00:00
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
2021-01-12 00:13:58 +00:00
pairs=expanded_pairs, timeframes=config['timeframes'],
2020-08-26 18:52:09 +00:00
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
2021-01-12 00:13:58 +00:00
)
2020-01-26 12:17:26 +00:00
else:
if not exchange.get_option('ohlcv_has_history', True):
raise OperationalException(
f"Historic klines not available for {exchange.name}. "
"Please use `--dl-trades` instead for this exchange "
"(will unfortunately take a long time)."
)
migrate_binance_futures_data(config)
2020-01-26 12:17:26 +00:00
pairs_not_available = refresh_backtest_ohlcv_data(
2021-01-12 00:13:58 +00:00
exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange,
new_pairs_days=config['new_pairs_days'],
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
trading_mode=config.get('trading_mode', 'spot'),
2022-04-30 15:24:57 +00:00
prepend=config.get('prepend_data', False)
2022-02-12 22:10:21 +00:00
)
2020-01-26 12:17:26 +00:00
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")
finally:
if pairs_not_available:
logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
f"on exchange {exchange.name}.")
def start_convert_trades(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
timerange = TimeRange()
# Remove stake-currency to skip checks which are not relevant for datadownload
config['stake_currency'] = ''
if 'pairs' not in config:
raise OperationalException(
"Downloading data requires a list of pairs. "
"Please check the documentation on how to configure this.")
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
# Manual validations of relevant settings
if not config['exchange'].get('skip_pair_validation', False):
exchange.validate_pairs(config['pairs'])
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
logger.info(f"About to Convert pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
for timeframe in config['timeframes']:
exchange.validate_timeframes(timeframe)
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
)
def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
"""
Convert data from one format to another
"""
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if ohlcv:
migrate_binance_futures_data(config)
candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', ['spot'])]
for candle_type in candle_types:
convert_ohlcv_format(config,
convert_from=args['format_from'], convert_to=args['format_to'],
erase=args['erase'], candle_type=candle_type)
else:
convert_trades_format(config,
convert_from=args['format_from'], convert_to=args['format_to'],
erase=args['erase'])
2020-07-12 07:57:00 +00:00
def start_list_data(args: Dict[str, Any]) -> None:
"""
List available backtest data
"""
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
from tabulate import tabulate
2020-09-28 17:39:41 +00:00
from freqtrade.data.history.idatahandler import get_datahandler
2020-07-12 08:23:09 +00:00
dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
2022-03-03 06:06:13 +00:00
paircombs = dhc.ohlcv_get_available_data(
config['datadir'],
config.get('trading_mode', TradingMode.SPOT)
)
2020-07-12 07:57:00 +00:00
2020-07-14 04:55:34 +00:00
if args['pairs']:
paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]
2020-07-12 07:57:00 +00:00
print(f"Found {len(paircombs)} pair / timeframe combinations.")
2022-08-19 11:44:31 +00:00
if not config.get('show_timerange'):
groupedpair = defaultdict(list)
for pair, timeframe, candle_type in sorted(
paircombs,
key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
):
groupedpair[(pair, candle_type)].append(timeframe)
if groupedpair:
print(tabulate([
(pair, ', '.join(timeframes), candle_type)
for (pair, candle_type), timeframes in groupedpair.items()
],
headers=("Pair", "Timeframe", "Type"),
tablefmt='psql', stralign='right'))
else:
paircombs1 = [(
pair, timeframe, candle_type,
*dhc.ohlcv_data_min_max(pair, timeframe, candle_type)
) for pair, timeframe, candle_type in paircombs]
print(tabulate([
2022-08-19 11:44:31 +00:00
(pair, timeframe, candle_type,
start.strftime(DATETIME_PRINT_FORMAT),
end.strftime(DATETIME_PRINT_FORMAT))
for pair, timeframe, candle_type, start, end in sorted(
paircombs1,
key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]))
2022-08-19 11:44:31 +00:00
],
headers=("Pair", "Timeframe", "Type", 'From', 'To'),
tablefmt='psql', stralign='right'))