Cache pairlist in pairlist, not globally

closes #4797 closes #4689
This commit is contained in:
Matthias 2021-04-25 20:10:47 +02:00
parent 7448a05f15
commit bb7ef2f804
5 changed files with 19 additions and 36 deletions

View File

@ -73,7 +73,7 @@ class IPairList(LoggingMixin, ABC):
"""
raise NotImplementedError()
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
def gen_pairlist(self, tickers: Dict) -> List[str]:
"""
Generate the pairlist.
@ -84,7 +84,6 @@ class IPairList(LoggingMixin, ABC):
it will raise the exception if a Pairlist Handler is used at the first
position in the chain.
:param cached_pairlist: Previously generated pairlist (cached)
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs
"""

View File

@ -42,10 +42,9 @@ class StaticPairList(IPairList):
"""
return f"{self.name}"
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
def gen_pairlist(self, tickers: Dict) -> List[str]:
"""
Generate the pairlist
:param cached_pairlist: Previously generated pairlist (cached)
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs
"""

View File

@ -4,9 +4,10 @@ Volume PairList provider
Provides dynamic pair list based on trade volumes
"""
import logging
from datetime import datetime
from typing import Any, Dict, List
from cachetools.ttl import TTLCache
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList
@ -33,7 +34,8 @@ class VolumePairList(IPairList):
self._number_pairs = self._pairlistconfig['number_assets']
self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume')
self._min_value = self._pairlistconfig.get('min_value', 0)
self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
if not self._exchange.exchange_has('fetchTickers'):
raise OperationalException(
@ -63,17 +65,19 @@ class VolumePairList(IPairList):
"""
return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
def gen_pairlist(self, tickers: Dict) -> List[str]:
"""
Generate the pairlist
:param cached_pairlist: Previously generated pairlist (cached)
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs
"""
# Generate dynamic whitelist
# Must always run if this pairlist is not the first in the list.
if self._last_refresh + self.refresh_period < datetime.now().timestamp():
self._last_refresh = int(datetime.now().timestamp())
pairlist = self._pair_cache.get('pairlist')
if pairlist:
# Item found - no refresh necessary
return pairlist
else:
# Use fresh pairlist
# Check if pair quote currency equals to the stake currency.
@ -82,9 +86,9 @@ class VolumePairList(IPairList):
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
and v[self._sort_key] is not None)]
pairlist = [s['symbol'] for s in filtered_tickers]
else:
# Use the cached pairlist if it's not time yet to refresh
pairlist = cached_pairlist
pairlist = self.filter_pairlist(pairlist, tickers)
self._pair_cache['pairlist'] = pairlist
return pairlist

View File

@ -3,7 +3,7 @@ PairList manager class
"""
import logging
from copy import deepcopy
from typing import Any, Dict, List
from typing import Dict, List
from cachetools import TTLCache, cached
@ -79,11 +79,8 @@ class PairListManager():
if self._tickers_needed:
tickers = self._get_cached_tickers()
# Adjust whitelist if filters are using tickers
pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
# Generate the pairlist with first Pairlist Handler in the chain
pairlist = self._pairlist_handlers[0].gen_pairlist(pairlist, tickers)
pairlist = self._pairlist_handlers[0].gen_pairlist(tickers)
# Process all Pairlist Handlers in the chain
for pairlist_handler in self._pairlist_handlers:
@ -95,19 +92,6 @@ class PairListManager():
self._whitelist = pairlist
def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]:
"""
Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove
pairs that do not have ticker available
"""
if self._tickers_needed:
# Copy list since we're modifying this list
for p in deepcopy(pairlist):
if p not in tickers:
pairlist.remove(p)
return pairlist
def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
"""
Verify and remove items from pairlist - returning a filtered pairlist.

View File

@ -604,17 +604,14 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):
get_tickers=tickers
)
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == 0
assert len(freqtrade.pairlists._pairlist_handlers[0]._pair_cache) == 0
assert tickers.call_count == 0
freqtrade.pairlists.refresh_pairlist()
assert tickers.call_count == 1
assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh != 0
lrf = freqtrade.pairlists._pairlist_handlers[0]._last_refresh
assert len(freqtrade.pairlists._pairlist_handlers[0]._pair_cache) == 1
freqtrade.pairlists.refresh_pairlist()
assert tickers.call_count == 1
# Time should not be updated.
assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf
def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers):