Cache pairlist in pairlist, not globally

closes #4797 closes #4689
This commit is contained in:
Matthias 2021-04-25 20:10:47 +02:00
parent 7448a05f15
commit bb7ef2f804
5 changed files with 19 additions and 36 deletions

View File

@ -73,7 +73,7 @@ class IPairList(LoggingMixin, ABC):
""" """
raise NotImplementedError() raise NotImplementedError()
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: def gen_pairlist(self, tickers: Dict) -> List[str]:
""" """
Generate the pairlist. Generate the pairlist.
@ -84,7 +84,6 @@ class IPairList(LoggingMixin, ABC):
it will raise the exception if a Pairlist Handler is used at the first it will raise the exception if a Pairlist Handler is used at the first
position in the chain. position in the chain.
:param cached_pairlist: Previously generated pairlist (cached)
:param tickers: Tickers (from exchange.get_tickers()). May be cached. :param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs :return: List of pairs
""" """

View File

@ -42,10 +42,9 @@ class StaticPairList(IPairList):
""" """
return f"{self.name}" return f"{self.name}"
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: def gen_pairlist(self, tickers: Dict) -> List[str]:
""" """
Generate the pairlist Generate the pairlist
:param cached_pairlist: Previously generated pairlist (cached)
:param tickers: Tickers (from exchange.get_tickers()). May be cached. :param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs :return: List of pairs
""" """

View File

@ -4,9 +4,10 @@ Volume PairList provider
Provides dynamic pair list based on trade volumes Provides dynamic pair list based on trade volumes
""" """
import logging import logging
from datetime import datetime
from typing import Any, Dict, List from typing import Any, Dict, List
from cachetools.ttl import TTLCache
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.plugins.pairlist.IPairList import IPairList
@ -33,7 +34,8 @@ class VolumePairList(IPairList):
self._number_pairs = self._pairlistconfig['number_assets'] self._number_pairs = self._pairlistconfig['number_assets']
self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume')
self._min_value = self._pairlistconfig.get('min_value', 0) self._min_value = self._pairlistconfig.get('min_value', 0)
self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
if not self._exchange.exchange_has('fetchTickers'): if not self._exchange.exchange_has('fetchTickers'):
raise OperationalException( raise OperationalException(
@ -63,17 +65,19 @@ class VolumePairList(IPairList):
""" """
return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: def gen_pairlist(self, tickers: Dict) -> List[str]:
""" """
Generate the pairlist Generate the pairlist
:param cached_pairlist: Previously generated pairlist (cached)
:param tickers: Tickers (from exchange.get_tickers()). May be cached. :param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs :return: List of pairs
""" """
# Generate dynamic whitelist # Generate dynamic whitelist
# Must always run if this pairlist is not the first in the list. # Must always run if this pairlist is not the first in the list.
if self._last_refresh + self.refresh_period < datetime.now().timestamp(): pairlist = self._pair_cache.get('pairlist')
self._last_refresh = int(datetime.now().timestamp()) if pairlist:
# Item found - no refresh necessary
return pairlist
else:
# Use fresh pairlist # Use fresh pairlist
# Check if pair quote currency equals to the stake currency. # Check if pair quote currency equals to the stake currency.
@ -82,9 +86,9 @@ class VolumePairList(IPairList):
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
and v[self._sort_key] is not None)] and v[self._sort_key] is not None)]
pairlist = [s['symbol'] for s in filtered_tickers] pairlist = [s['symbol'] for s in filtered_tickers]
else:
# Use the cached pairlist if it's not time yet to refresh pairlist = self.filter_pairlist(pairlist, tickers)
pairlist = cached_pairlist self._pair_cache['pairlist'] = pairlist
return pairlist return pairlist

View File

@ -3,7 +3,7 @@ PairList manager class
""" """
import logging import logging
from copy import deepcopy from copy import deepcopy
from typing import Any, Dict, List from typing import Dict, List
from cachetools import TTLCache, cached from cachetools import TTLCache, cached
@ -79,11 +79,8 @@ class PairListManager():
if self._tickers_needed: if self._tickers_needed:
tickers = self._get_cached_tickers() tickers = self._get_cached_tickers()
# Adjust whitelist if filters are using tickers
pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
# Generate the pairlist with first Pairlist Handler in the chain # Generate the pairlist with first Pairlist Handler in the chain
pairlist = self._pairlist_handlers[0].gen_pairlist(pairlist, tickers) pairlist = self._pairlist_handlers[0].gen_pairlist(tickers)
# Process all Pairlist Handlers in the chain # Process all Pairlist Handlers in the chain
for pairlist_handler in self._pairlist_handlers: for pairlist_handler in self._pairlist_handlers:
@ -95,19 +92,6 @@ class PairListManager():
self._whitelist = pairlist self._whitelist = pairlist
def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]:
"""
Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove
pairs that do not have ticker available
"""
if self._tickers_needed:
# Copy list since we're modifying this list
for p in deepcopy(pairlist):
if p not in tickers:
pairlist.remove(p)
return pairlist
def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]: def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
""" """
Verify and remove items from pairlist - returning a filtered pairlist. Verify and remove items from pairlist - returning a filtered pairlist.

View File

@ -604,17 +604,14 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):
get_tickers=tickers get_tickers=tickers
) )
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == 0 assert len(freqtrade.pairlists._pairlist_handlers[0]._pair_cache) == 0
assert tickers.call_count == 0 assert tickers.call_count == 0
freqtrade.pairlists.refresh_pairlist() freqtrade.pairlists.refresh_pairlist()
assert tickers.call_count == 1 assert tickers.call_count == 1
assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh != 0 assert len(freqtrade.pairlists._pairlist_handlers[0]._pair_cache) == 1
lrf = freqtrade.pairlists._pairlist_handlers[0]._last_refresh
freqtrade.pairlists.refresh_pairlist() freqtrade.pairlists.refresh_pairlist()
assert tickers.call_count == 1 assert tickers.call_count == 1
# Time should not be updated.
assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf
def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers): def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers):