Merge pull request #7663 from freqtrade/shuffle_list_enhance

Improve ShufflePairlist to shuffle only once per candle

docs/includes/pairlists.md

@@ -286,6 +286,18 @@ Min price precision for SHITCOIN/BTC is 8 decimals. If its price is 0.00000011 -
 
 Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority.
 
+By default, ShuffleFilter will shuffle pairs once per candle.
+To shuffle on every iteration, set `"shuffle_frequency"` to `"iteration"` instead of  the default of `"candle"`.
+
+``` json
+    {
+        "method": "ShuffleFilter", 
+        "shuffle_frequency": "candle",
+        "seed": 42
+    }
+
+```
+
 !!! Tip
     You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. ShuffleFilter will automatically detect runmodes and apply the `seed` only for backtesting modes - if a `seed` value is set.
 

freqtrade/plugins/pairlist/IPairList.py

@@ -36,7 +36,6 @@ class IPairList(LoggingMixin, ABC):
         self._pairlistconfig = pairlistconfig
         self._pairlist_pos = pairlist_pos
         self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
-        self._last_refresh = 0
         LoggingMixin.__init__(self, logger, self.refresh_period)
 
     @property

freqtrade/plugins/pairlist/ShuffleFilter.py

@@ -3,16 +3,20 @@ Shuffle pair list filter
 """
 import logging
 import random
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Literal
 
 from freqtrade.constants import Config
 from freqtrade.enums import RunMode
+from freqtrade.exchange import timeframe_to_seconds
 from freqtrade.exchange.types import Tickers
 from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.util.periodic_cache import PeriodicCache
 
 
 logger = logging.getLogger(__name__)
 
+ShuffleValues = Literal['candle', 'iteration']
+
 
 class ShuffleFilter(IPairList):
 
@@ -31,6 +35,9 @@ class ShuffleFilter(IPairList):
             logger.info(f"Backtesting mode detected, applying seed value: {self._seed}")
 
         self._random = random.Random(self._seed)
+        self._shuffle_freq: ShuffleValues = pairlistconfig.get('shuffle_frequency', 'candle')
+        self.__pairlist_cache = PeriodicCache(
+                    maxsize=1000, ttl=timeframe_to_seconds(self._config['timeframe']))
 
     @property
     def needstickers(self) -> bool:
@@ -45,7 +52,7 @@ class ShuffleFilter(IPairList):
         """
         Short whitelist method description - used for startup-messages
         """
-        return (f"{self.name} - Shuffling pairs" +
+        return (f"{self.name} - Shuffling pairs every {self._shuffle_freq}" +
                 (f", seed = {self._seed}." if self._seed is not None else "."))
 
     def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]:
@@ -56,7 +63,13 @@ class ShuffleFilter(IPairList):
         :param tickers: Tickers (from exchange.get_tickers). May be cached.
         :return: new whitelist
         """
+        pairlist_bef = tuple(pairlist)
+        pairlist_new = self.__pairlist_cache.get(pairlist_bef)
+        if pairlist_new and self._shuffle_freq == 'candle':
+            # Use cached pairlist.
+            return pairlist_new
         # Shuffle is done inplace
         self._random.shuffle(pairlist)
+        self.__pairlist_cache[pairlist_bef] = pairlist
 
         return pairlist

tests/plugins/test_pairlist.py

@@ -2,6 +2,8 @@
 
 import logging
 import time
+from copy import deepcopy
+from datetime import timedelta
 from unittest.mock import MagicMock, PropertyMock
 
 import pandas as pd
@@ -719,15 +721,26 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
 def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None:
     whitelist_conf['pairlists'] = [
         {"method": "StaticPairList"},
-        {"method": "ShuffleFilter", "seed": 42}
+        {"method": "ShuffleFilter", "seed": 43}
     ]
 
     exchange = get_patched_exchange(mocker, whitelist_conf)
-    PairListManager(exchange, whitelist_conf)
-    assert log_has("Backtesting mode detected, applying seed value: 42", caplog)
+    plm = PairListManager(exchange, whitelist_conf)
+    assert log_has("Backtesting mode detected, applying seed value: 43", caplog)
+
+    with time_machine.travel("2021-09-01 05:01:00 +00:00") as t:
+        plm.refresh_pairlist()
+        pl1 = deepcopy(plm.whitelist)
+        plm.refresh_pairlist()
+        assert plm.whitelist == pl1
+
+        t.shift(timedelta(minutes=10))
+        plm.refresh_pairlist()
+        assert plm.whitelist != pl1
+
     caplog.clear()
     whitelist_conf['runmode'] = RunMode.DRY_RUN
-    PairListManager(exchange, whitelist_conf)
+    plm = PairListManager(exchange, whitelist_conf)
     assert not log_has("Backtesting mode detected, applying seed value: 42", caplog)
     assert log_has("Live mode detected, not applying seed.", caplog)