From 3300d25e574a748538153b6f9bf336c257ae418b Mon Sep 17 00:00:00 2001 From: liuweiqing Date: Fri, 8 Nov 2024 14:50:21 +0800 Subject: [PATCH 1/6] feat: enhance data fetching logic with pagination to increase market cap rank limit --- freqtrade/plugins/pairlist/MarketCapPairList.py | 11 ++++++++--- tests/plugins/test_pairlist.py | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index 2394e910e..874e7bb2c 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -5,6 +5,7 @@ Provides dynamic pair list based on Market Cap """ import logging +import math from cachetools import TTLCache @@ -56,8 +57,8 @@ class MarketCapPairList(IPairList): f"You can choose from {category_ids}" ) - if self._max_rank > 250: - raise OperationalException("This filter only support marketcap rank up to 250.") + if self._max_rank > 1000: + raise OperationalException("This filter only support marketcap rank up to 1000.") @property def needstickers(self) -> bool: @@ -165,7 +166,11 @@ class MarketCapPairList(IPairList): data = [] if not self._categories: - data = self._coingecko.get_coins_markets(**default_kwargs) + pages_required = math.ceil(self._max_rank / 250) + for page in range(1, pages_required + 1): + default_kwargs["page"] = page + page_data = self._coingecko.get_coins_markets(**default_kwargs) + data.extend(page_data) else: for category in self._categories: category_data = self._coingecko.get_coins_markets( diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 1c138cc55..0cb1df323 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -2458,10 +2458,10 @@ def test_MarketCapPairList_exceptions(mocker, default_conf_usdt): PairListManager(exchange, default_conf_usdt) default_conf_usdt["pairlists"] = [ - {"method": "MarketCapPairList", "number_assets": 20, "max_rank": 260} + {"method": "MarketCapPairList", "number_assets": 20, "max_rank": 1010} ] with pytest.raises( - OperationalException, match="This filter only support marketcap rank up to 250." + OperationalException, match="This filter only support marketcap rank up to 1000." ): PairListManager(exchange, default_conf_usdt) From 00318be59fe0711cb31fa21f843b162033cb661e Mon Sep 17 00:00:00 2001 From: liuweiqing Date: Sat, 9 Nov 2024 11:25:31 +0800 Subject: [PATCH 2/6] chore: Convert page number to string to ensure type compatibility --- freqtrade/plugins/pairlist/MarketCapPairList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index 874e7bb2c..b925d0b21 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -168,7 +168,7 @@ class MarketCapPairList(IPairList): if not self._categories: pages_required = math.ceil(self._max_rank / 250) for page in range(1, pages_required + 1): - default_kwargs["page"] = page + default_kwargs["page"] = str(page) page_data = self._coingecko.get_coins_markets(**default_kwargs) data.extend(page_data) else: From 7b471d59c5da9f1456715f1d4c6eef2672f44e30 Mon Sep 17 00:00:00 2001 From: liuweiqing Date: Sat, 9 Nov 2024 18:23:27 +0800 Subject: [PATCH 3/6] chore: add warning when max_rank exceeds 250 in MarketCapPairList --- freqtrade/plugins/pairlist/MarketCapPairList.py | 8 ++++++-- tests/plugins/test_pairlist.py | 10 ++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index b925d0b21..01a724529 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -6,6 +6,7 @@ Provides dynamic pair list based on Market Cap import logging import math +import warnings from cachetools import TTLCache @@ -57,8 +58,11 @@ class MarketCapPairList(IPairList): f"You can choose from {category_ids}" ) - if self._max_rank > 1000: - raise OperationalException("This filter only support marketcap rank up to 1000.") + if self._max_rank > 250: + self.logger.warning( + f"The max rank you have set ({self._max_rank}) is quite high. " + "Please ensure this value is appropriate for your use case.", + ) @property def needstickers(self) -> bool: diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 0cb1df323..a4f66a702 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -2450,7 +2450,7 @@ def test_MarketCapPairList_filter_special_no_pair_from_coingecko( assert pm.whitelist == [] -def test_MarketCapPairList_exceptions(mocker, default_conf_usdt): +def test_MarketCapPairList_exceptions(mocker, default_conf_usdt, caplog): exchange = get_patched_exchange(mocker, default_conf_usdt) default_conf_usdt["pairlists"] = [{"method": "MarketCapPairList"}] with pytest.raises(OperationalException, match=r"`number_assets` not specified.*"): @@ -2458,13 +2458,11 @@ def test_MarketCapPairList_exceptions(mocker, default_conf_usdt): PairListManager(exchange, default_conf_usdt) default_conf_usdt["pairlists"] = [ - {"method": "MarketCapPairList", "number_assets": 20, "max_rank": 1010} + {"method": "MarketCapPairList", "number_assets": 20, "max_rank": 500} ] - with pytest.raises( - OperationalException, match="This filter only support marketcap rank up to 1000." - ): + with caplog.at_level(logging.WARNING): PairListManager(exchange, default_conf_usdt) - + assert log_has_re("The max rank you have set \\(500\\) is quite high", caplog) # Test invalid coinmarkets list mocker.patch( "freqtrade.plugins.pairlist.MarketCapPairList.FtCoinGeckoApi.get_coins_categories_list", From 27d894b8e9d28843b294dd103fadc22cb3705943 Mon Sep 17 00:00:00 2001 From: liuweiqing Date: Sat, 9 Nov 2024 18:40:32 +0800 Subject: [PATCH 4/6] chore: remove warnings --- freqtrade/plugins/pairlist/MarketCapPairList.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index 01a724529..cd17f8325 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -6,7 +6,6 @@ Provides dynamic pair list based on Market Cap import logging import math -import warnings from cachetools import TTLCache From b7a2efcd6a49b7dd260026743d446c6eef942308 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2024 18:12:48 +0100 Subject: [PATCH 5/6] docs: update docs with new marketcaplist behavior --- docs/includes/pairlists.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 8d79a7bc1..7e4659807 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -352,7 +352,7 @@ The optional `bearer_token` will be included in the requests Authorization Heade #### MarketCapPairList -`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The returned pairlist will be sorted based of their marketcap ranks. +`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. The returned pairlist will be sorted based of their marketcap ranks. ```json "pairlists": [ @@ -366,7 +366,8 @@ The optional `bearer_token` will be included in the requests Authorization Heade ] ``` -`number_assets` defines the maximum number of pairs returned by the pairlist. `max_rank` will determine the maximum rank used in creating/filtering the pairlist. It's expected that some coins within the top `max_rank` marketcap will not be included in the resulting pairlist since not all pairs will have active trading pairs in your preferred market/stake/exchange combination. +`number_assets` defines the maximum number of pairs returned by the pairlist. `max_rank` will determine the maximum rank used in creating/filtering the pairlist. It's expected that some coins within the top `max_rank` marketcap will not be included in the resulting pairlist since not all pairs will have active trading pairs in your preferred market/stake/exchange combination. +While using a `max_rank` bigger than 250 is supported, it's not recommended, as it'll cause multiple API calls to CoinGecko, which can lead to rate limit issues. The `refresh_period` setting defines the interval (in seconds) at which the marketcap rank data will be refreshed. The default is 86,400 seconds (1 day). The pairlist cache (`refresh_period`) applies to both generating pairlists (when in the first position in the list) and filtering instances (when not in the first position in the list). From 3a81b6a08f83864c63f2010a92ec5e638d24af60 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2024 18:18:54 +0100 Subject: [PATCH 6/6] chore: improve warning wording --- freqtrade/plugins/pairlist/MarketCapPairList.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index cd17f8325..b95cec50f 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -60,7 +60,8 @@ class MarketCapPairList(IPairList): if self._max_rank > 250: self.logger.warning( f"The max rank you have set ({self._max_rank}) is quite high. " - "Please ensure this value is appropriate for your use case.", + "This may lead to coingecko API rate limit issues. " + "Please ensure this value is necessary for your use case.", ) @property