From 83861fabdea798100c5322a9efe5db4f7a23c84a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Nov 2020 10:52:15 +0100 Subject: [PATCH 1/2] Fix #3967, move TradeList type to constants --- freqtrade/constants.py | 3 +++ freqtrade/data/converter.py | 10 +++++---- freqtrade/data/history/hdf5datahandler.py | 4 ++-- freqtrade/data/history/idatahandler.py | 5 +---- freqtrade/data/history/jsondatahandler.py | 4 ++-- tests/data/test_converter.py | 27 ++++++++++++++++++++++- 6 files changed, 40 insertions(+), 13 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index dc5384f6f..3271dda39 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -365,3 +365,6 @@ CANCEL_REASON = { # List of pairs with their timeframes PairWithTimeframe = Tuple[str, str] ListPairsWithTimeframes = List[PairWithTimeframe] + +# Type for trades list +TradeList = List[List] diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 38fa670e9..09930950a 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List import pandas as pd from pandas import DataFrame, to_datetime -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList logger = logging.getLogger(__name__) @@ -168,7 +168,7 @@ def trades_remove_duplicates(trades: List[List]) -> List[List]: return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))] -def trades_dict_to_list(trades: List[Dict]) -> List[List]: +def trades_dict_to_list(trades: List[Dict]) -> TradeList: """ Convert fetch_trades result into a List (to be more memory efficient). :param trades: List of trades, as returned by ccxt.fetch_trades. @@ -177,16 +177,18 @@ def trades_dict_to_list(trades: List[Dict]) -> List[List]: return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades] -def trades_to_ohlcv(trades: List, timeframe: str) -> DataFrame: +def trades_to_ohlcv(trades: TradeList, timeframe: str) -> DataFrame: """ Converts trades list to OHLCV list - TODO: This should get a dedicated test :param trades: List of trades, as returned by ccxt.fetch_trades. :param timeframe: Timeframe to resample data to :return: OHLCV Dataframe. + :raises: Valueerror if no trades are provided """ from freqtrade.exchange import timeframe_to_minutes timeframe_minutes = timeframe_to_minutes(timeframe) + if not trades: + raise ValueError('Trade-list empty.') df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS) df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True,) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 00e41673d..d116637e7 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -9,9 +9,9 @@ import pandas as pd from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, - ListPairsWithTimeframes) + ListPairsWithTimeframes, TradeList) -from .idatahandler import IDataHandler, TradeList +from .idatahandler import IDataHandler logger = logging.getLogger(__name__) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index a170a9dc5..070d9039d 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -13,16 +13,13 @@ from typing import List, Optional, Type from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.constants import ListPairsWithTimeframes, TradeList from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe from freqtrade.exchange import timeframe_to_seconds logger = logging.getLogger(__name__) -# Type for trades list -TradeList = List[List] - class IDataHandler(ABC): diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 6436aa13d..9122170d5 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -8,10 +8,10 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes, TradeList from freqtrade.data.converter import trades_dict_to_list -from .idatahandler import IDataHandler, TradeList +from .idatahandler import IDataHandler logger = logging.getLogger(__name__) diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index fdba7900f..4fdcce4d2 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -1,10 +1,13 @@ # pragma pylint: disable=missing-docstring, C0103 import logging +import pytest + from freqtrade.configuration.timerange import TimeRange from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, ohlcv_fill_up_missing_data, ohlcv_to_dataframe, - trades_dict_to_list, trades_remove_duplicates, trim_dataframe) + trades_dict_to_list, trades_remove_duplicates, + trades_to_ohlcv, trim_dataframe) from freqtrade.data.history import (get_timerange, load_data, load_pair_history, validate_backtest_data) from tests.conftest import log_has @@ -26,6 +29,28 @@ def test_ohlcv_to_dataframe(ohlcv_history_list, caplog): assert log_has('Converting candle (OHLCV) data to dataframe for pair UNITTEST/BTC.', caplog) +def test_trades_to_ohlcv(ohlcv_history_list, caplog): + + caplog.set_level(logging.DEBUG) + with pytest.raises(ValueError, match="Trade-list empty."): + trades_to_ohlcv([], '1m') + + trades = [ + [1570752011620, "13519807", None, "sell", 0.00141342, 23.0, 0.03250866], + [1570752011620, "13519808", None, "sell", 0.00141266, 54.0, 0.07628364], + [1570752017964, "13519809", None, "sell", 0.00141266, 8.0, 0.01130128]] + + df = trades_to_ohlcv(trades, '1m') + assert not df.empty + assert len(df) == 1 + assert 'open' in df.columns + assert 'high' in df.columns + assert 'low' in df.columns + assert 'close' in df.columns + assert df.loc[:, 'high'][0] == 0.00141342 + assert df.loc[:, 'low'][0] == 0.00141266 + + def test_ohlcv_fill_up_missing_data(testdatadir, caplog): data = load_pair_history(datadir=testdatadir, timeframe='1m', From e8e3ca0c3c114637a460fdddeea5fca155bdf534 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Nov 2020 10:57:19 +0100 Subject: [PATCH 2/2] Catch ValueError from trade_conversion closes #3967 --- freqtrade/data/converter.py | 2 +- freqtrade/data/history/history_utils.py | 9 ++++++--- tests/data/test_history.py | 6 ++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 09930950a..d4053abaa 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -183,7 +183,7 @@ def trades_to_ohlcv(trades: TradeList, timeframe: str) -> DataFrame: :param trades: List of trades, as returned by ccxt.fetch_trades. :param timeframe: Timeframe to resample data to :return: OHLCV Dataframe. - :raises: Valueerror if no trades are provided + :raises: ValueError if no trades are provided """ from freqtrade.exchange import timeframe_to_minutes timeframe_minutes = timeframe_to_minutes(timeframe) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index a420b9dcc..17b510b92 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -356,9 +356,12 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], if erase: if data_handler_ohlcv.ohlcv_purge(pair, timeframe): logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') - ohlcv = trades_to_ohlcv(trades, timeframe) - # Store ohlcv - data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv) + try: + ohlcv = trades_to_ohlcv(trades, timeframe) + # Store ohlcv + data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv) + except ValueError: + logger.exception(f'Could not convert {pair} to OHLCV.') def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 538a0840f..905798041 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -620,6 +620,12 @@ def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): _clean_test_file(file1) _clean_test_file(file5) + assert not log_has('Could not convert NoDatapair to OHLCV.', caplog) + + convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], + datadir=testdatadir, timerange=tr, erase=True) + assert log_has('Could not convert NoDatapair to OHLCV.', caplog) + def test_datahandler_ohlcv_get_pairs(testdatadir): pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m')