Merge pull request #10902 from 14790897/develop

feat: enhance data fetching logic with pagination to increase market …

docs/includes/pairlists.md

@@ -352,7 +352,7 @@ The optional `bearer_token` will be included in the requests Authorization Heade
 
 #### MarketCapPairList
 
-`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The returned pairlist will be sorted based of their marketcap ranks.
+`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. The returned pairlist will be sorted based of their marketcap ranks.
 
 ```json
 "pairlists": [
@@ -366,7 +366,8 @@ The optional `bearer_token` will be included in the requests Authorization Heade
 ]
 ```
 
-`number_assets` defines the maximum number of pairs returned by the pairlist. `max_rank` will determine the maximum rank used in creating/filtering the pairlist. It's expected that some coins within the top `max_rank` marketcap will not be included in the resulting pairlist since not all pairs will have active trading pairs in your preferred market/stake/exchange combination.
+`number_assets` defines the maximum number of pairs returned by the pairlist. `max_rank` will determine the maximum rank used in creating/filtering the pairlist. It's expected that some coins within the top `max_rank` marketcap will not be included in the resulting pairlist since not all pairs will have active trading pairs in your preferred market/stake/exchange combination.  
+While using a `max_rank` bigger than 250 is supported, it's not recommended, as it'll cause multiple API calls to CoinGecko, which can lead to rate limit issues.
 
 The `refresh_period` setting defines the interval (in seconds) at which the marketcap rank data will be refreshed. The default is 86,400 seconds (1 day). The pairlist cache (`refresh_period`) applies to both generating pairlists (when in the first position in the list) and filtering instances (when not in the first position in the list).
 

freqtrade/plugins/pairlist/MarketCapPairList.py

@@ -5,6 +5,7 @@ Provides dynamic pair list based on Market Cap
 """
 
 import logging
+import math
 
 from cachetools import TTLCache
 
@@ -57,7 +58,11 @@ class MarketCapPairList(IPairList):
                     )
 
         if self._max_rank > 250:
-            raise OperationalException("This filter only support marketcap rank up to 250.")
+            self.logger.warning(
+                f"The max rank you have set ({self._max_rank}) is quite high. "
+                "This may lead to coingecko API rate limit issues. "
+                "Please ensure this value is necessary for your use case.",
+            )
 
     @property
     def needstickers(self) -> bool:
@@ -165,7 +170,11 @@ class MarketCapPairList(IPairList):
             data = []
 
             if not self._categories:
-                data = self._coingecko.get_coins_markets(**default_kwargs)
+                pages_required = math.ceil(self._max_rank / 250)
+                for page in range(1, pages_required + 1):
+                    default_kwargs["page"] = str(page)
+                    page_data = self._coingecko.get_coins_markets(**default_kwargs)
+                    data.extend(page_data)
             else:
                 for category in self._categories:
                     category_data = self._coingecko.get_coins_markets(

tests/plugins/test_pairlist.py

@@ -2450,7 +2450,7 @@ def test_MarketCapPairList_filter_special_no_pair_from_coingecko(
     assert pm.whitelist == []
 
 
-def test_MarketCapPairList_exceptions(mocker, default_conf_usdt):
+def test_MarketCapPairList_exceptions(mocker, default_conf_usdt, caplog):
     exchange = get_patched_exchange(mocker, default_conf_usdt)
     default_conf_usdt["pairlists"] = [{"method": "MarketCapPairList"}]
     with pytest.raises(OperationalException, match=r"`number_assets` not specified.*"):
@@ -2458,13 +2458,11 @@ def test_MarketCapPairList_exceptions(mocker, default_conf_usdt):
         PairListManager(exchange, default_conf_usdt)
 
     default_conf_usdt["pairlists"] = [
-        {"method": "MarketCapPairList", "number_assets": 20, "max_rank": 260}
+        {"method": "MarketCapPairList", "number_assets": 20, "max_rank": 500}
     ]
-    with pytest.raises(
+    with caplog.at_level(logging.WARNING):
-        OperationalException, match="This filter only support marketcap rank up to 250."
-    ):
         PairListManager(exchange, default_conf_usdt)
-
+    assert log_has_re("The max rank you have set \\(500\\) is quite high", caplog)
     # Test invalid coinmarkets list
     mocker.patch(
         "freqtrade.plugins.pairlist.MarketCapPairList.FtCoinGeckoApi.get_coins_categories_list",