Refactor ohlcv caching to exchange class

freqtrade/exchange/exchange.py

@@ -6,6 +6,7 @@ import asyncio
 import inspect
 import logging
 import signal
+from collections import defaultdict
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from math import floor
@@ -48,6 +49,7 @@ from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.util import dt_from_ts, dt_now
 from freqtrade.util.datetime_helpers import dt_humanize, dt_ts
+from freqtrade.util.periodic_cache import PeriodicCache
 
 
 logger = logging.getLogger(__name__)
@@ -138,6 +140,7 @@ class Exchange:
 
         # Holds candles
         self._klines: Dict[PairWithTimeframe, DataFrame] = {}
+        self._expiring_candle_cache: Dict[str, PeriodicCache] = {}
 
         # Holds public_trades
         self._trades: Dict[PairWithTimeframe, DataFrame] = {}
@@ -2267,6 +2270,39 @@ class Exchange:
 
         return results_df
 
+    def refresh_ohlcv_with_cache(
+        self,
+        pairs: List[PairWithTimeframe],
+        since_ms: int
+    ) -> Dict[PairWithTimeframe, DataFrame]:
+        """
+        Refresh ohlcv data for all pairs in needed_pairs if necessary.
+        Caches data with expiring per timeframe.
+        Should only be used for pairlists which need "on time" expirarion, and no longer cache.
+        """
+
+        timeframes = [p[1] for p in pairs]
+        for timeframe in timeframes:
+            if timeframe not in self._expiring_candle_cache:
+                timeframe_in_sec = timeframe_to_seconds(timeframe)
+                # Initialise cache
+                self._expiring_candle_cache[timeframe] = PeriodicCache(ttl=timeframe_in_sec,
+                                                                       maxsize=1000)
+
+        # Get candles from cache
+        candles = {
+            c: self._expiring_candle_cache[c[1]].get(c, None) for c in pairs
+            if c in self._expiring_candle_cache[c[1]]
+        }
+        pairs_to_download = [p for p in pairs if p not in candles]
+        if pairs_to_download:
+            candles = self.refresh_latest_ohlcv(
+                pairs_to_download, since_ms=since_ms, cache=False
+            )
+            for c, val in candles.items():
+                self._expiring_candle_cache[c[1]][c] = val
+        return candles
+
     def _now_is_time_to_refresh(self, pair: str, timeframe: str, candle_type: CandleType) -> bool:
         # Timeframe in seconds
         interval_in_sec = timeframe_to_seconds(timeframe)

freqtrade/plugins/pairlist/VolumePairList.py

@@ -14,7 +14,7 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.exchange.types import Tickers
 from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
-from freqtrade.util import PeriodicCache, dt_now, format_ms_time
+from freqtrade.util import dt_now, format_ms_time
 
 
 logger = logging.getLogger(__name__)
@@ -63,7 +63,6 @@ class VolumePairList(IPairList):
         # get timeframe in minutes and seconds
         self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
         _tf_in_sec = self._tf_in_min * 60
-        self._candle_cache = PeriodicCache(maxsize=1000, ttl=_tf_in_sec)
 
         # wether to use range lookback or not
         self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
@@ -230,18 +229,7 @@ class VolumePairList(IPairList):
                 if p not in self._pair_cache
             ]
 
-            # Get all candles
+            candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms)
-            candles = {
-                c: self._candle_cache.get(c, None) for c in needed_pairs
-                if c in self._candle_cache
-            }
-            pairs_to_download = [p for p in needed_pairs if p not in candles]
-            if pairs_to_download:
-                candles = self._exchange.refresh_latest_ohlcv(
-                    pairs_to_download, since_ms=since_ms, cache=False
-                )
-                for c, val in candles.items():
-                    self._candle_cache[c] = val
 
             for i, p in enumerate(filtered_tickers):
                 contract_size = self._exchange.markets[p['symbol']].get('contractSize', 1.0) or 1.0