diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 7a896e495..bccc6ea9a 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -5,8 +5,9 @@ from typing import Any, Dict from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Config -from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format -from freqtrade.data.history import convert_trades_to_ohlcv, download_data_main +from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, + convert_trades_to_ohlcv) +from freqtrade.data.history import download_data_main from freqtrade.enums import RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 7d9014e87..5a503faad 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -2,12 +2,14 @@ Functions to convert data from one format to another """ import logging +from pathlib import Path from typing import Dict, List import numpy as np import pandas as pd from pandas import DataFrame, to_datetime +from freqtrade.configuration import TimeRange from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES, Config, TradeList) from freqtrade.enums import CandleType, TradingMode @@ -260,6 +262,42 @@ def trades_to_ohlcv(trades: DataFrame, timeframe: str) -> DataFrame: return df_new.loc[:, DEFAULT_DATAFRAME_COLUMNS] +def convert_trades_to_ohlcv( + pairs: List[str], + timeframes: List[str], + datadir: Path, + timerange: TimeRange, + erase: bool = False, + data_format_ohlcv: str = 'feather', + data_format_trades: str = 'feather', + candle_type: CandleType = CandleType.SPOT +) -> None: + """ + Convert stored trades data to ohlcv data + """ + from freqtrade.data.history.idatahandler import get_datahandler + data_handler_trades = get_datahandler(datadir, data_format=data_format_trades) + data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv) + if not pairs: + pairs = data_handler_trades.trades_get_pairs(datadir) + + logger.info(f"About to convert pairs: '{', '.join(pairs)}', " + f"intervals: '{', '.join(timeframes)}' to {datadir}") + + for pair in pairs: + trades = data_handler_trades.trades_load(pair) + for timeframe in timeframes: + if erase: + if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type): + logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') + try: + ohlcv = trades_to_ohlcv(trades, timeframe) + # Store ohlcv + data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type) + except ValueError: + logger.exception(f'Could not convert {pair} to OHLCV.') + + def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool): """ Convert trades from one format to another format. diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index b95b48b90..1ad1060a4 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -9,9 +9,9 @@ from pandas import DataFrame, concat from freqtrade.configuration import TimeRange from freqtrade.constants import (DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS, DL_DATA_TIMEFRAMES, Config) -from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, - trades_df_remove_duplicates, trades_list_to_df, - trades_to_ohlcv) +from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_ohlcv, + ohlcv_to_dataframe, trades_df_remove_duplicates, + trades_list_to_df) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException @@ -429,41 +429,6 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: return pairs_not_available -def convert_trades_to_ohlcv( - pairs: List[str], - timeframes: List[str], - datadir: Path, - timerange: TimeRange, - erase: bool = False, - data_format_ohlcv: str = 'feather', - data_format_trades: str = 'feather', - candle_type: CandleType = CandleType.SPOT -) -> None: - """ - Convert stored trades data to ohlcv data - """ - data_handler_trades = get_datahandler(datadir, data_format=data_format_trades) - data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv) - if not pairs: - pairs = data_handler_trades.trades_get_pairs(datadir) - - logger.info(f"About to convert pairs: '{', '.join(pairs)}', " - f"intervals: '{', '.join(timeframes)}' to {datadir}") - - for pair in pairs: - trades = data_handler_trades.trades_load(pair) - for timeframe in timeframes: - if erase: - if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type): - logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') - try: - ohlcv = trades_to_ohlcv(trades, timeframe) - # Store ohlcv - data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type) - except ValueError: - logger.exception(f'Could not convert {pair} to OHLCV.') - - def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]: """ Get the maximum common timerange for the given backtest data.