freqtrade_origin/tests/data/test_dataprovider.py

615 lines
22 KiB
Python
Raw Normal View History

2020-06-14 09:51:20 +00:00
from datetime import datetime, timezone
2020-05-12 04:38:14 +00:00
from unittest.mock import MagicMock
2018-12-25 12:56:30 +00:00
import pytest
2022-12-10 19:03:42 +00:00
from pandas import DataFrame, Timestamp
2018-12-25 12:56:30 +00:00
from freqtrade.data.dataprovider import DataProvider
2021-12-08 14:59:20 +00:00
from freqtrade.enums import CandleType, RunMode
from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.plugins.pairlistmanager import PairListManager
from tests.conftest import EXMS, generate_test_data, get_patched_exchange
2018-12-25 12:56:30 +00:00
2024-05-12 14:00:45 +00:00
@pytest.mark.parametrize(
"candle_type",
[
"mark",
"",
],
)
2021-12-07 06:11:36 +00:00
def test_dp_ohlcv(mocker, default_conf, ohlcv_history, candle_type):
2019-01-22 05:56:54 +00:00
default_conf["runmode"] = RunMode.DRY_RUN
2020-06-01 18:47:27 +00:00
timeframe = default_conf["timeframe"]
2018-12-25 12:56:30 +00:00
exchange = get_patched_exchange(mocker, default_conf)
2021-12-07 06:11:36 +00:00
candletype = CandleType.from_string(candle_type)
2021-12-08 13:35:15 +00:00
exchange._klines[("XRP/BTC", timeframe, candletype)] = ohlcv_history
exchange._klines[("UNITTEST/BTC", timeframe, candletype)] = ohlcv_history
2018-12-25 12:56:30 +00:00
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.DRY_RUN
2021-12-07 06:11:36 +00:00
assert ohlcv_history.equals(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype))
assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype), DataFrame)
assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype) is not ohlcv_history
assert dp.ohlcv("UNITTEST/BTC", timeframe, copy=False, candle_type=candletype) is ohlcv_history
assert not dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype).empty
assert dp.ohlcv("NONSENSE/AAA", timeframe, candle_type=candletype).empty
2018-12-30 06:15:49 +00:00
# Test with and without parameter
2024-05-12 14:00:45 +00:00
assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype).equals(
dp.ohlcv("UNITTEST/BTC", candle_type=candle_type)
)
2019-01-22 05:56:54 +00:00
default_conf["runmode"] = RunMode.LIVE
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.LIVE
assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame)
2019-01-22 05:56:54 +00:00
default_conf["runmode"] = RunMode.BACKTEST
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.BACKTEST
assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type).empty
2018-12-25 12:56:30 +00:00
def test_historic_ohlcv(mocker, default_conf, ohlcv_history):
historymock = MagicMock(return_value=ohlcv_history)
2019-01-22 05:56:54 +00:00
mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock)
2018-12-25 12:56:30 +00:00
dp = DataProvider(default_conf, None)
2019-01-22 05:56:54 +00:00
data = dp.historic_ohlcv("UNITTEST/BTC", "5m")
2018-12-25 12:56:30 +00:00
assert isinstance(data, DataFrame)
assert historymock.call_count == 1
assert historymock.call_args_list[0][1]["timeframe"] == "5m"
2018-12-26 13:23:21 +00:00
2024-04-05 05:16:37 +00:00
def test_historic_trades(mocker, default_conf, trades_history_df):
historymock = MagicMock(return_value=trades_history_df)
mocker.patch(
"freqtrade.data.history.datahandlers.featherdatahandler.FeatherDataHandler._trades_load",
2024-05-15 15:09:32 +00:00
historymock,
2024-04-05 05:16:37 +00:00
)
dp = DataProvider(default_conf, None)
# Live mode..
with pytest.raises(OperationalException, match=r"Exchange is not available to DataProvider\."):
dp.trades("UNITTEST/BTC", "5m")
exchange = get_patched_exchange(mocker, default_conf)
dp = DataProvider(default_conf, exchange)
data = dp.trades("UNITTEST/BTC", "5m")
assert isinstance(data, DataFrame)
assert len(data) == 0
# Switch to backtest mode
2024-05-15 15:09:32 +00:00
default_conf["runmode"] = RunMode.BACKTEST
default_conf["dataformat_trades"] = "feather"
2024-04-05 05:16:37 +00:00
exchange = get_patched_exchange(mocker, default_conf)
dp = DataProvider(default_conf, exchange)
data = dp.trades("UNITTEST/BTC", "5m")
assert isinstance(data, DataFrame)
assert len(data) == len(trades_history_df)
def test_historic_ohlcv_dataformat(mocker, default_conf, ohlcv_history):
hdf5loadmock = MagicMock(return_value=ohlcv_history)
2023-07-09 15:06:11 +00:00
featherloadmock = MagicMock(return_value=ohlcv_history)
2024-03-15 05:49:49 +00:00
mocker.patch(
"freqtrade.data.history.datahandlers.hdf5datahandler.HDF5DataHandler._ohlcv_load",
2024-05-12 14:00:45 +00:00
hdf5loadmock,
)
2024-03-15 05:49:49 +00:00
mocker.patch(
"freqtrade.data.history.datahandlers.featherdatahandler.FeatherDataHandler._ohlcv_load",
2024-05-12 14:00:45 +00:00
featherloadmock,
)
default_conf["runmode"] = RunMode.BACKTEST
exchange = get_patched_exchange(mocker, default_conf)
dp = DataProvider(default_conf, exchange)
data = dp.historic_ohlcv("UNITTEST/BTC", "5m")
assert isinstance(data, DataFrame)
hdf5loadmock.assert_not_called()
2023-07-09 15:06:11 +00:00
featherloadmock.assert_called_once()
2021-08-16 12:16:24 +00:00
# Switching to dataformat hdf5
hdf5loadmock.reset_mock()
2023-07-09 15:06:11 +00:00
featherloadmock.reset_mock()
default_conf["dataformat_ohlcv"] = "hdf5"
dp = DataProvider(default_conf, exchange)
data = dp.historic_ohlcv("UNITTEST/BTC", "5m")
assert isinstance(data, DataFrame)
hdf5loadmock.assert_called_once()
2023-07-09 15:06:11 +00:00
featherloadmock.assert_not_called()
2024-05-12 14:00:45 +00:00
@pytest.mark.parametrize(
"candle_type",
[
"mark",
"futures",
"",
],
)
def test_get_pair_dataframe(mocker, default_conf, ohlcv_history, candle_type):
2019-08-17 09:59:27 +00:00
default_conf["runmode"] = RunMode.DRY_RUN
2020-06-01 18:47:27 +00:00
timeframe = default_conf["timeframe"]
2019-08-17 09:59:27 +00:00
exchange = get_patched_exchange(mocker, default_conf)
2021-12-07 06:11:36 +00:00
candletype = CandleType.from_string(candle_type)
2021-12-08 13:35:15 +00:00
exchange._klines[("XRP/BTC", timeframe, candletype)] = ohlcv_history
exchange._klines[("UNITTEST/BTC", timeframe, candletype)] = ohlcv_history
2019-08-17 09:59:27 +00:00
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.DRY_RUN
2024-05-12 14:00:45 +00:00
assert ohlcv_history.equals(
dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type)
)
assert ohlcv_history.equals(
dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candletype)
)
assert isinstance(
dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame
)
assert (
dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type)
is not ohlcv_history
)
assert not dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type).empty
assert dp.get_pair_dataframe("NONSENSE/AAA", timeframe, candle_type=candle_type).empty
2019-08-17 09:59:27 +00:00
# Test with and without parameter
2024-05-12 14:00:45 +00:00
assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type).equals(
dp.get_pair_dataframe("UNITTEST/BTC", candle_type=candle_type)
)
2019-08-17 09:59:27 +00:00
default_conf["runmode"] = RunMode.LIVE
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.LIVE
2024-05-12 14:00:45 +00:00
assert isinstance(
dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame
)
assert dp.get_pair_dataframe("NONSENSE/AAA", timeframe, candle_type=candle_type).empty
2019-08-17 09:59:27 +00:00
historymock = MagicMock(return_value=ohlcv_history)
2019-08-17 09:59:27 +00:00
mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock)
default_conf["runmode"] = RunMode.BACKTEST
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.BACKTEST
df = dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type)
assert isinstance(df, DataFrame)
assert len(df) == 3 # ohlcv_history mock has just 3 rows
2024-05-12 14:00:45 +00:00
dp._set_dataframe_max_date(ohlcv_history.iloc[-1]["date"])
df = dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type)
assert isinstance(df, DataFrame)
assert len(df) == 2 # ohlcv_history is limited to 2 rows now
2019-08-17 09:59:27 +00:00
def test_available_pairs(mocker, default_conf, ohlcv_history):
2018-12-26 13:23:21 +00:00
exchange = get_patched_exchange(mocker, default_conf)
2020-06-01 18:47:27 +00:00
timeframe = default_conf["timeframe"]
exchange._klines[("XRP/BTC", timeframe)] = ohlcv_history
exchange._klines[("UNITTEST/BTC", timeframe)] = ohlcv_history
2018-12-26 13:23:21 +00:00
dp = DataProvider(default_conf, exchange)
2018-12-26 13:23:21 +00:00
assert len(dp.available_pairs) == 2
2024-05-12 14:00:45 +00:00
assert dp.available_pairs == [
("XRP/BTC", timeframe),
("UNITTEST/BTC", timeframe),
]
2019-01-22 05:55:40 +00:00
2022-12-16 05:46:44 +00:00
def test_producer_pairs(default_conf):
2022-09-09 23:14:40 +00:00
dataprovider = DataProvider(default_conf, None)
producer = "default"
whitelist = ["XRP/BTC", "ETH/BTC"]
assert len(dataprovider.get_producer_pairs(producer)) == 0
dataprovider._set_producer_pairs(whitelist, producer)
assert len(dataprovider.get_producer_pairs(producer)) == 2
new_whitelist = ["BTC/USDT"]
dataprovider._set_producer_pairs(new_whitelist, producer)
assert dataprovider.get_producer_pairs(producer) == new_whitelist
assert dataprovider.get_producer_pairs("bad") == []
2022-12-16 05:46:44 +00:00
def test_get_producer_df(default_conf):
2022-09-09 23:14:40 +00:00
dataprovider = DataProvider(default_conf, None)
2024-05-12 14:00:45 +00:00
ohlcv_history = generate_test_data("5m", 150)
pair = "BTC/USDT"
timeframe = default_conf["timeframe"]
2022-09-09 23:14:40 +00:00
candle_type = CandleType.SPOT
empty_la = datetime.fromtimestamp(0, tz=timezone.utc)
2022-09-12 22:39:16 +00:00
now = datetime.now(timezone.utc)
2022-09-09 23:14:40 +00:00
# no data has been added, any request should return an empty dataframe
dataframe, la = dataprovider.get_producer_df(pair, timeframe, candle_type)
2022-09-09 23:14:40 +00:00
assert dataframe.empty
assert la == empty_la
# the data is added, should return that added dataframe
2022-09-12 22:39:16 +00:00
dataprovider._add_external_df(pair, ohlcv_history, now, timeframe, candle_type)
dataframe, la = dataprovider.get_producer_df(pair, timeframe, candle_type)
2022-09-09 23:14:40 +00:00
assert len(dataframe) > 0
assert la > empty_la
# no data on this producer, should return empty dataframe
2024-05-12 14:00:45 +00:00
dataframe, la = dataprovider.get_producer_df(pair, producer_name="bad")
2022-09-09 23:14:40 +00:00
assert dataframe.empty
assert la == empty_la
# non existent timeframe, empty dataframe
2024-05-12 14:00:45 +00:00
_dataframe, la = dataprovider.get_producer_df(pair, timeframe="1h")
2022-09-09 23:14:40 +00:00
assert dataframe.empty
assert la == empty_la
def test_emit_df(mocker, default_conf, ohlcv_history):
2024-05-12 14:00:45 +00:00
mocker.patch("freqtrade.rpc.rpc_manager.RPCManager.__init__", MagicMock())
rpc_mock = mocker.patch("freqtrade.rpc.rpc_manager.RPCManager", MagicMock())
send_mock = mocker.patch("freqtrade.rpc.rpc_manager.RPCManager.send_msg", MagicMock())
2022-09-09 23:14:40 +00:00
dataprovider = DataProvider(default_conf, exchange=None, rpc=rpc_mock)
dataprovider_no_rpc = DataProvider(default_conf, exchange=None)
pair = "BTC/USDT"
# No emit yet
assert send_mock.call_count == 0
# Rpc is added, we call emit, should call send_msg
2022-10-26 18:31:31 +00:00
dataprovider._emit_df(pair, ohlcv_history, False)
2022-09-09 23:14:40 +00:00
assert send_mock.call_count == 1
2022-10-26 18:31:31 +00:00
send_mock.reset_mock()
dataprovider._emit_df(pair, ohlcv_history, True)
assert send_mock.call_count == 2
send_mock.reset_mock()
2022-09-09 23:14:40 +00:00
# No rpc added, emit called, should not call send_msg
2022-10-26 18:31:31 +00:00
dataprovider_no_rpc._emit_df(pair, ohlcv_history, False)
assert send_mock.call_count == 0
2022-09-09 23:14:40 +00:00
2022-12-16 05:46:44 +00:00
def test_refresh(mocker, default_conf):
2024-04-05 05:07:12 +00:00
refresh_mock = mocker.patch(f"{EXMS}.refresh_latest_ohlcv")
mock_refresh_trades = mocker.patch(f"{EXMS}.refresh_latest_trades")
2019-01-22 05:55:40 +00:00
exchange = get_patched_exchange(mocker, default_conf, exchange="binance")
2020-06-01 18:47:27 +00:00
timeframe = default_conf["timeframe"]
pairs = [("XRP/BTC", timeframe), ("UNITTEST/BTC", timeframe)]
2019-01-22 05:55:40 +00:00
pairs_non_trad = [("ETH/USDT", timeframe), ("BTC/TUSD", "1h")]
2019-01-22 05:55:40 +00:00
dp = DataProvider(default_conf, exchange)
dp.refresh(pairs)
2024-04-05 05:07:12 +00:00
assert mock_refresh_trades.call_count == 0
2019-01-22 05:55:40 +00:00
assert refresh_mock.call_count == 1
assert len(refresh_mock.call_args[0]) == 1
assert len(refresh_mock.call_args[0][0]) == len(pairs)
assert refresh_mock.call_args[0][0] == pairs
refresh_mock.reset_mock()
dp.refresh(pairs, pairs_non_trad)
2024-04-05 05:07:12 +00:00
assert mock_refresh_trades.call_count == 0
2019-01-22 05:55:40 +00:00
assert refresh_mock.call_count == 1
assert len(refresh_mock.call_args[0]) == 1
assert len(refresh_mock.call_args[0][0]) == len(pairs) + len(pairs_non_trad)
assert refresh_mock.call_args[0][0] == pairs + pairs_non_trad
2024-04-05 05:07:12 +00:00
# Test with public trades
refresh_mock.reset_mock()
refresh_mock.reset_mock()
2024-05-15 15:09:32 +00:00
default_conf["exchange"]["use_public_trades"] = True
2024-04-05 05:07:12 +00:00
dp.refresh(pairs, pairs_non_trad)
assert mock_refresh_trades.call_count == 1
assert refresh_mock.call_count == 1
def test_orderbook(mocker, default_conf, order_book_l2):
api_mock = MagicMock()
api_mock.fetch_l2_order_book = order_book_l2
exchange = get_patched_exchange(mocker, default_conf, api_mock=api_mock)
dp = DataProvider(default_conf, exchange)
2024-05-12 14:00:45 +00:00
res = dp.orderbook("ETH/BTC", 5)
assert order_book_l2.call_count == 1
2024-05-12 14:00:45 +00:00
assert order_book_l2.call_args_list[0][0][0] == "ETH/BTC"
assert order_book_l2.call_args_list[0][0][1] >= 5
2023-08-22 18:39:36 +00:00
assert isinstance(res, dict)
2024-05-12 14:00:45 +00:00
assert "bids" in res
assert "asks" in res
def test_market(mocker, default_conf, markets):
api_mock = MagicMock()
api_mock.markets = markets
exchange = get_patched_exchange(mocker, default_conf, api_mock=api_mock)
dp = DataProvider(default_conf, exchange)
2024-05-12 14:00:45 +00:00
res = dp.market("ETH/BTC")
2023-08-22 18:39:36 +00:00
assert isinstance(res, dict)
2024-05-12 14:00:45 +00:00
assert "symbol" in res
assert res["symbol"] == "ETH/BTC"
2024-05-12 14:00:45 +00:00
res = dp.market("UNITTEST/BTC")
assert res is None
2020-05-15 16:19:52 +00:00
def test_ticker(mocker, default_conf, tickers):
2024-05-12 14:00:45 +00:00
ticker_mock = MagicMock(return_value=tickers()["ETH/BTC"])
mocker.patch(f"{EXMS}.fetch_ticker", ticker_mock)
2020-05-15 16:19:52 +00:00
exchange = get_patched_exchange(mocker, default_conf)
2020-05-14 13:43:16 +00:00
dp = DataProvider(default_conf, exchange)
2024-05-12 14:00:45 +00:00
res = dp.ticker("ETH/BTC")
2023-08-22 18:39:36 +00:00
assert isinstance(res, dict)
2024-05-12 14:00:45 +00:00
assert "symbol" in res
assert res["symbol"] == "ETH/BTC"
2020-05-14 13:43:16 +00:00
2024-05-12 14:00:45 +00:00
ticker_mock = MagicMock(side_effect=ExchangeError("Pair not found"))
mocker.patch(f"{EXMS}.fetch_ticker", ticker_mock)
2020-05-15 16:19:52 +00:00
exchange = get_patched_exchange(mocker, default_conf)
dp = DataProvider(default_conf, exchange)
2024-05-12 14:00:45 +00:00
res = dp.ticker("UNITTEST/BTC")
2020-05-14 13:43:16 +00:00
assert res == {}
2020-05-12 04:38:14 +00:00
def test_current_whitelist(mocker, default_conf, tickers):
# patch default conf to volumepairlist
2024-05-12 14:00:45 +00:00
default_conf["pairlists"][0] = {"method": "VolumePairList", "number_assets": 5}
2024-05-12 14:00:45 +00:00
mocker.patch.multiple(EXMS, exchange_has=MagicMock(return_value=True), get_tickers=tickers)
2020-05-12 04:38:14 +00:00
exchange = get_patched_exchange(mocker, default_conf)
pairlist = PairListManager(exchange, default_conf)
dp = DataProvider(default_conf, exchange, pairlist)
# Simulate volumepairs from exchange.
2020-05-12 04:38:14 +00:00
pairlist.refresh_pairlist()
assert dp.current_whitelist() == pairlist._whitelist
2021-04-13 04:57:21 +00:00
# The identity of the 2 lists should not be identical, but a copy
assert dp.current_whitelist() is not pairlist._whitelist
with pytest.raises(OperationalException):
dp = DataProvider(default_conf, exchange)
dp.current_whitelist()
2020-06-14 09:51:20 +00:00
def test_get_analyzed_dataframe(mocker, default_conf, ohlcv_history):
default_conf["runmode"] = RunMode.DRY_RUN
timeframe = default_conf["timeframe"]
exchange = get_patched_exchange(mocker, default_conf)
dp = DataProvider(default_conf, exchange)
2021-12-03 12:04:31 +00:00
dp._set_cached_df("XRP/BTC", timeframe, ohlcv_history, CandleType.SPOT)
dp._set_cached_df("UNITTEST/BTC", timeframe, ohlcv_history, CandleType.SPOT)
2020-06-14 09:51:20 +00:00
assert dp.runmode == RunMode.DRY_RUN
dataframe, time = dp.get_analyzed_dataframe("UNITTEST/BTC", timeframe)
assert ohlcv_history.equals(dataframe)
assert isinstance(time, datetime)
dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe)
assert ohlcv_history.equals(dataframe)
assert isinstance(time, datetime)
dataframe, time = dp.get_analyzed_dataframe("NOTHING/BTC", timeframe)
assert dataframe.empty
assert isinstance(time, datetime)
assert time == datetime(1970, 1, 1, tzinfo=timezone.utc)
2021-05-05 18:08:31 +00:00
2021-05-09 17:47:37 +00:00
# Test backtest mode
default_conf["runmode"] = RunMode.BACKTEST
dp._set_dataframe_max_index(1)
dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe)
assert len(dataframe) == 1
dp._set_dataframe_max_index(2)
dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe)
assert len(dataframe) == 2
dp._set_dataframe_max_index(3)
dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe)
assert len(dataframe) == 3
dp._set_dataframe_max_index(500)
dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe)
assert len(dataframe) == len(ohlcv_history)
2021-05-05 18:08:31 +00:00
def test_no_exchange_mode(default_conf):
dp = DataProvider(default_conf, None)
message = "Exchange is not available to DataProvider."
with pytest.raises(OperationalException, match=message):
dp.refresh([()])
with pytest.raises(OperationalException, match=message):
2024-05-12 14:00:45 +00:00
dp.ohlcv("XRP/USDT", "5m", "")
2021-05-05 18:08:31 +00:00
with pytest.raises(OperationalException, match=message):
2024-05-12 14:00:45 +00:00
dp.market("XRP/USDT")
2021-05-05 18:08:31 +00:00
with pytest.raises(OperationalException, match=message):
2024-05-12 14:00:45 +00:00
dp.ticker("XRP/USDT")
2021-05-05 18:08:31 +00:00
with pytest.raises(OperationalException, match=message):
2024-05-12 14:00:45 +00:00
dp.orderbook("XRP/USDT", 20)
2021-05-09 17:47:37 +00:00
with pytest.raises(OperationalException, match=message):
dp.available_pairs()
def test_dp_send_msg(default_conf):
default_conf["runmode"] = RunMode.DRY_RUN
2024-05-12 14:00:45 +00:00
default_conf["timeframe"] = "1h"
dp = DataProvider(default_conf, None)
2024-05-12 14:00:45 +00:00
msg = "Test message"
dp.send_msg(msg)
assert msg in dp._msg_queue
dp._msg_queue.pop()
assert msg not in dp._msg_queue
# Message is not resent due to caching
dp.send_msg(msg)
assert msg not in dp._msg_queue
dp.send_msg(msg, always_send=True)
assert msg in dp._msg_queue
default_conf["runmode"] = RunMode.BACKTEST
dp = DataProvider(default_conf, None)
dp.send_msg(msg, always_send=True)
assert msg not in dp._msg_queue
2022-12-10 19:03:42 +00:00
2022-12-14 18:58:45 +00:00
def test_dp__add_external_df(default_conf_usdt):
2024-05-12 14:00:45 +00:00
timeframe = "1h"
2022-12-10 19:03:42 +00:00
default_conf_usdt["timeframe"] = timeframe
dp = DataProvider(default_conf_usdt, None)
2024-05-12 14:00:45 +00:00
df = generate_test_data(timeframe, 24, "2022-01-01 00:00:00+00:00")
2022-12-10 19:03:42 +00:00
last_analyzed = datetime.now(timezone.utc)
2024-05-12 14:00:45 +00:00
res = dp._add_external_df("ETH/USDT", df, last_analyzed, timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
assert res[0] is False
# Why 1000 ??
assert res[1] == 1000
2022-12-14 18:58:45 +00:00
# Hard add dataframe
2024-05-12 14:00:45 +00:00
dp._replace_external_df("ETH/USDT", df, last_analyzed, timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
# BTC is not stored yet
2024-05-12 14:00:45 +00:00
res = dp._add_external_df("BTC/USDT", df, last_analyzed, timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
assert res[0] is False
2024-05-12 14:00:45 +00:00
df_res, _ = dp.get_producer_df("ETH/USDT", timeframe, CandleType.SPOT)
2022-12-14 18:58:45 +00:00
assert len(df_res) == 24
2022-12-10 19:03:42 +00:00
# Add the same dataframe again - dataframe size shall not change.
2024-05-12 14:00:45 +00:00
res = dp._add_external_df("ETH/USDT", df, last_analyzed, timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
assert res[0] is True
assert isinstance(res[1], int)
2022-12-10 19:03:42 +00:00
assert res[1] == 0
2024-05-12 14:00:45 +00:00
df, _ = dp.get_producer_df("ETH/USDT", timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
assert len(df) == 24
# Add a new day.
2024-05-12 14:00:45 +00:00
df2 = generate_test_data(timeframe, 24, "2022-01-02 00:00:00+00:00")
2022-12-10 19:03:42 +00:00
2024-05-12 14:00:45 +00:00
res = dp._add_external_df("ETH/USDT", df2, last_analyzed, timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
assert res[0] is True
assert isinstance(res[1], int)
2022-12-10 19:03:42 +00:00
assert res[1] == 0
2024-05-12 14:00:45 +00:00
df, _ = dp.get_producer_df("ETH/USDT", timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
assert len(df) == 48
# Add a dataframe with a 12 hour offset - so 12 candles are overlapping, and 12 valid.
2024-05-12 14:00:45 +00:00
df3 = generate_test_data(timeframe, 24, "2022-01-02 12:00:00+00:00")
2022-12-10 19:03:42 +00:00
2024-05-12 14:00:45 +00:00
res = dp._add_external_df("ETH/USDT", df3, last_analyzed, timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
assert res[0] is True
assert isinstance(res[1], int)
2022-12-10 19:03:42 +00:00
assert res[1] == 0
2024-05-12 14:00:45 +00:00
df, _ = dp.get_producer_df("ETH/USDT", timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
# New length = 48 + 12 (since we have a 12 hour offset).
assert len(df) == 60
2024-05-12 14:00:45 +00:00
assert df.iloc[-1]["date"] == df3.iloc[-1]["date"]
assert df.iloc[-1]["date"] == Timestamp("2022-01-03 11:00:00+00:00")
2022-12-10 19:03:42 +00:00
# Generate 1 new candle
2024-05-12 14:00:45 +00:00
df4 = generate_test_data(timeframe, 1, "2022-01-03 12:00:00+00:00")
res = dp._add_external_df("ETH/USDT", df4, last_analyzed, timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
# assert res[0] is True
# assert res[1] == 0
2024-05-12 14:00:45 +00:00
df, _ = dp.get_producer_df("ETH/USDT", timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
# New length = 61 + 1
assert len(df) == 61
2024-05-12 14:00:45 +00:00
assert df.iloc[-2]["date"] == Timestamp("2022-01-03 11:00:00+00:00")
assert df.iloc[-1]["date"] == Timestamp("2022-01-03 12:00:00+00:00")
2022-12-10 19:03:42 +00:00
# Gap in the data ...
2024-05-12 14:00:45 +00:00
df4 = generate_test_data(timeframe, 1, "2022-01-05 00:00:00+00:00")
res = dp._add_external_df("ETH/USDT", df4, last_analyzed, timeframe, CandleType.SPOT)
2022-12-10 19:03:42 +00:00
assert res[0] is False
# 36 hours - from 2022-01-03 12:00:00+00:00 to 2022-01-05 00:00:00+00:00
assert isinstance(res[1], int)
2022-12-10 19:03:42 +00:00
assert res[1] == 36
2024-05-12 14:00:45 +00:00
df, _ = dp.get_producer_df("ETH/USDT", timeframe, CandleType.SPOT)
2022-12-11 07:49:35 +00:00
# New length = 61 + 1
assert len(df) == 61
# Empty dataframe
2024-05-12 14:00:45 +00:00
df4 = generate_test_data(timeframe, 0, "2022-01-05 00:00:00+00:00")
res = dp._add_external_df("ETH/USDT", df4, last_analyzed, timeframe, CandleType.SPOT)
2022-12-11 07:49:35 +00:00
assert res[0] is False
# 36 hours - from 2022-01-03 12:00:00+00:00 to 2022-01-05 00:00:00+00:00
assert isinstance(res[1], int)
2022-12-11 07:49:35 +00:00
assert res[1] == 0
2023-12-12 18:48:17 +00:00
def test_dp_get_required_startup(default_conf_usdt):
2024-05-12 14:00:45 +00:00
timeframe = "1h"
2023-12-12 18:48:17 +00:00
default_conf_usdt["timeframe"] = timeframe
dp = DataProvider(default_conf_usdt, None)
# No FreqAI config
2024-05-12 14:00:45 +00:00
assert dp.get_required_startup("5m") == 0
assert dp.get_required_startup("1h") == 0
assert dp.get_required_startup("1d") == 0
2023-12-12 18:48:17 +00:00
2024-05-12 14:00:45 +00:00
dp._config["startup_candle_count"] = 20
assert dp.get_required_startup("5m") == 20
assert dp.get_required_startup("1h") == 20
assert dp.get_required_startup("1h") == 20
2023-12-12 18:48:17 +00:00
# With freqAI config
2024-05-12 14:00:45 +00:00
dp._config["freqai"] = {
"enabled": True,
"train_period_days": 20,
"feature_parameters": {
"indicator_periods_candles": [
2023-12-12 18:48:17 +00:00
5,
20,
]
2024-05-12 14:00:45 +00:00
},
2023-12-12 18:48:17 +00:00
}
2024-05-12 14:00:45 +00:00
assert dp.get_required_startup("5m") == 5780
assert dp.get_required_startup("1h") == 500
assert dp.get_required_startup("1d") == 40
2023-12-12 18:48:17 +00:00
# FreqAI kindof ignores startup_candle_count if it's below indicator_periods_candles
2024-05-12 14:00:45 +00:00
dp._config["startup_candle_count"] = 0
assert dp.get_required_startup("5m") == 5780
assert dp.get_required_startup("1h") == 500
assert dp.get_required_startup("1d") == 40
2023-12-12 18:48:17 +00:00
2024-05-12 14:00:45 +00:00
dp._config["freqai"]["feature_parameters"]["indicator_periods_candles"][1] = 50
assert dp.get_required_startup("5m") == 5810
assert dp.get_required_startup("1h") == 530
assert dp.get_required_startup("1d") == 70
# scenario from issue https://github.com/freqtrade/freqtrade/issues/9432
2024-05-12 14:00:45 +00:00
dp._config["freqai"] = {
"enabled": True,
"train_period_days": 180,
"feature_parameters": {
"indicator_periods_candles": [
10,
20,
]
2024-05-12 14:00:45 +00:00
},
}
2024-05-12 14:00:45 +00:00
dp._config["startup_candle_count"] = 40
assert dp.get_required_startup("5m") == 51880
assert dp.get_required_startup("1h") == 4360
assert dp.get_required_startup("1d") == 220