freqtrade_origin/freqtrade/plugins/pairlist/RemotePairList.py

"""
Remote PairList provider

Provides pair list fetched from a remote source
"""

import logging
from pathlib import Path
from typing import Any, Dict, List, Tuple

import rapidjson
import requests
from cachetools import TTLCache

from freqtrade import __version__
from freqtrade.configuration.load_config import CONFIG_PARSE_MODE
from freqtrade.exceptions import OperationalException
from freqtrade.exchange.types import Tickers
from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist


logger = logging.getLogger(__name__)


class RemotePairList(IPairList):
    is_pairlist_generator = True
    # Potential winner bias
    supports_backtesting = SupportsBacktesting.TODAYS_DATA

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

        if "number_assets" not in self._pairlistconfig:
            raise OperationalException(
                "`number_assets` not specified. Please check your configuration "
                'for "pairlist.config.number_assets"'
            )

        if "pairlist_url" not in self._pairlistconfig:
            raise OperationalException(
                "`pairlist_url` not specified. Please check your configuration "
                'for "pairlist.config.pairlist_url"'
            )

        self._mode = self._pairlistconfig.get("mode", "whitelist")
        self._processing_mode = self._pairlistconfig.get("processing_mode", "filter")
        self._number_pairs = self._pairlistconfig["number_assets"]
        self._refresh_period: int = self._pairlistconfig.get("refresh_period", 1800)
        self._keep_pairlist_on_failure = self._pairlistconfig.get("keep_pairlist_on_failure", True)
        self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
        self._pairlist_url = self._pairlistconfig.get("pairlist_url", "")
        self._read_timeout = self._pairlistconfig.get("read_timeout", 60)
        self._bearer_token = self._pairlistconfig.get("bearer_token", "")
        self._init_done = False
        self._save_to_file = self._pairlistconfig.get("save_to_file", None)
        self._last_pairlist: List[Any] = list()

        if self._mode not in ["whitelist", "blacklist"]:
            raise OperationalException(
                "`mode` not configured correctly. Supported Modes " 'are "whitelist","blacklist"'
            )

        if self._processing_mode not in ["filter", "append"]:
            raise OperationalException(
                "`processing_mode` not configured correctly. Supported Modes "
                'are "filter","append"'
            )

        if self._pairlist_pos == 0 and self._mode == "blacklist":
            raise OperationalException(
                "A `blacklist` mode RemotePairList can not be on the first "
                "position of your pairlist."
            )

    @property
    def needstickers(self) -> bool:
        """
        Boolean property defining if tickers are necessary.
        If no Pairlist requires tickers, an empty Dict is passed
        as tickers argument to filter_pairlist
        """
        return False

    def short_desc(self) -> str:
        """
        Short whitelist method description - used for startup-messages
        """
        return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."

    @staticmethod
    def description() -> str:
        return "Retrieve pairs from a remote API or local file."

    @staticmethod
    def available_parameters() -> Dict[str, PairlistParameter]:
        return {
            "pairlist_url": {
                "type": "string",
                "default": "",
                "description": "URL to fetch pairlist from",
                "help": "URL to fetch pairlist from",
            },
            "number_assets": {
                "type": "number",
                "default": 30,
                "description": "Number of assets",
                "help": "Number of assets to use from the pairlist.",
            },
            "mode": {
                "type": "option",
                "default": "whitelist",
                "options": ["whitelist", "blacklist"],
                "description": "Pairlist mode",
                "help": "Should this pairlist operate as a whitelist or blacklist?",
            },
            "processing_mode": {
                "type": "option",
                "default": "filter",
                "options": ["filter", "append"],
                "description": "Processing mode",
                "help": "Append pairs to incoming pairlist or filter them?",
            },
            **IPairList.refresh_period_parameter(),
            "keep_pairlist_on_failure": {
                "type": "boolean",
                "default": True,
                "description": "Keep last pairlist on failure",
                "help": "Keep last pairlist on failure",
            },
            "read_timeout": {
                "type": "number",
                "default": 60,
                "description": "Read timeout",
                "help": "Request timeout for remote pairlist",
            },
            "bearer_token": {
                "type": "string",
                "default": "",
                "description": "Bearer token",
                "help": "Bearer token - used for auth against the upstream service.",
            },
            "save_to_file": {
                "type": "string",
                "default": "",
                "description": "Filename to save processed pairlist to.",
                "help": "Specify a filename to save the processed pairlist in JSON format.",
            },
        }

    def process_json(self, jsonparse) -> List[str]:
        pairlist = jsonparse.get("pairs", [])
        remote_refresh_period = int(jsonparse.get("refresh_period", self._refresh_period))

        if self._refresh_period < remote_refresh_period:
            self.log_once(
                f"Refresh Period has been increased from {self._refresh_period}"
                f" to minimum allowed: {remote_refresh_period} from Remote.",
                logger.info,
            )

            self._refresh_period = remote_refresh_period
            self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)

        self._init_done = True

        return pairlist

    def return_last_pairlist(self) -> List[str]:
        if self._keep_pairlist_on_failure:
            pairlist = self._last_pairlist
            self.log_once("Keeping last fetched pairlist", logger.info)
        else:
            pairlist = []

        return pairlist

    def fetch_pairlist(self) -> Tuple[List[str], float]:
        headers = {"User-Agent": "Freqtrade/" + __version__ + " Remotepairlist"}

        if self._bearer_token:
            headers["Authorization"] = f"Bearer {self._bearer_token}"

        try:
            response = requests.get(self._pairlist_url, headers=headers, timeout=self._read_timeout)
            content_type = response.headers.get("content-type")
            time_elapsed = response.elapsed.total_seconds()

            if "application/json" in str(content_type):
                jsonparse = response.json()

                try:
                    pairlist = self.process_json(jsonparse)
                except Exception as e:
                    pairlist = self._handle_error(f"Failed processing JSON data: {type(e)}")
            else:
                pairlist = self._handle_error(
                    f"RemotePairList is not of type JSON. {self._pairlist_url}"
                )

        except requests.exceptions.RequestException:
            pairlist = self._handle_error(
                f"Was not able to fetch pairlist from: {self._pairlist_url}"
            )

            time_elapsed = 0

        return pairlist, time_elapsed

    def _handle_error(self, error: str) -> List[str]:
        if self._init_done:
            self.log_once("Error: " + error, logger.info)
            return self.return_last_pairlist()
        else:
            raise OperationalException(error)

    def gen_pairlist(self, tickers: Tickers) -> List[str]:
        """
        Generate the pairlist
        :param tickers: Tickers (from exchange.get_tickers). May be cached.
        :return: List of pairs
        """

        if self._init_done:
            pairlist = self._pair_cache.get("pairlist")
            if pairlist == [None]:
                # Valid but empty pairlist.
                return []
        else:
            pairlist = []

        time_elapsed = 0.0

        if pairlist:
            # Item found - no refresh necessary
            return pairlist.copy()
        else:
            if self._pairlist_url.startswith("file:///"):
                filename = self._pairlist_url.split("file:///", 1)[1]
                file_path = Path(filename)

                if file_path.exists():
                    with file_path.open() as json_file:
                        try:
                            # Load the JSON data into a dictionary
                            jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE)
                            pairlist = self.process_json(jsonparse)
                        except Exception as e:
                            pairlist = self._handle_error(f"processing JSON data: {type(e)}")
                else:
                    pairlist = self._handle_error(f"{self._pairlist_url} does not exist.")

            else:
                # Fetch Pairlist from Remote URL
                pairlist, time_elapsed = self.fetch_pairlist()

        self.log_once(f"Fetched pairs: {pairlist}", logger.debug)

        pairlist = expand_pairlist(pairlist, list(self._exchange.get_markets().keys()))
        pairlist = self._whitelist_for_active_markets(pairlist)
        pairlist = pairlist[: self._number_pairs]

        if pairlist:
            self._pair_cache["pairlist"] = pairlist.copy()
        else:
            # If pairlist is empty, set a dummy value to avoid fetching again
            self._pair_cache["pairlist"] = [None]

        if time_elapsed != 0.0:
            self.log_once(f"Pairlist Fetched in {time_elapsed} seconds.", logger.info)
        else:
            self.log_once("Fetched Pairlist.", logger.info)

        self._last_pairlist = list(pairlist)

        if self._save_to_file:
            self.save_pairlist(pairlist, self._save_to_file)

        return pairlist

    def save_pairlist(self, pairlist: List[str], filename: str) -> None:
        pairlist_data = {"pairs": pairlist}
        try:
            file_path = Path(filename)
            with file_path.open("w") as json_file:
                rapidjson.dump(pairlist_data, json_file)
                logger.info(f"Processed pairlist saved to {filename}")
        except Exception as e:
            logger.error(f"Error saving processed pairlist to {filename}: {e}")

    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
        """
        Filters and sorts pairlist and returns the whitelist again.
        Called on each bot iteration - please use internal caching if necessary
        :param pairlist: pairlist to filter or sort
        :param tickers: Tickers (from exchange.get_tickers). May be cached.
        :return: new whitelist
        """
        rpl_pairlist = self.gen_pairlist(tickers)
        merged_list = []
        filtered = []

        if self._mode == "whitelist":
            if self._processing_mode == "filter":
                merged_list = [pair for pair in pairlist if pair in rpl_pairlist]
            elif self._processing_mode == "append":
                merged_list = pairlist + rpl_pairlist
            merged_list = sorted(set(merged_list), key=merged_list.index)
        else:
            for pair in pairlist:
                if pair not in rpl_pairlist:
                    merged_list.append(pair)
                else:
                    filtered.append(pair)
            if filtered:
                self.log_once(f"Blacklist - Filtered out pairs: {filtered}", logger.info)

        merged_list = merged_list[: self._number_pairs]
        return merged_list