From fd9c02603cfea41876dd562e224e15025c0637a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 15:59:52 +0100 Subject: [PATCH] Introduce chainable PairlistFilters --- freqtrade/pairlist/IPairList.py | 36 +++++++++-- freqtrade/pairlist/IPairListFilter.py | 18 ++++++ freqtrade/pairlist/LowPriceFilter.py | 48 ++++++++++++++ freqtrade/pairlist/PrecisionFilter.py | 51 +++++++++++++++ freqtrade/pairlist/StaticPairList.py | 2 +- freqtrade/pairlist/VolumePairList.py | 62 +------------------ .../resolvers/pairlistfilter_resolver.py | 53 ++++++++++++++++ 7 files changed, 203 insertions(+), 67 deletions(-) create mode 100644 freqtrade/pairlist/IPairListFilter.py create mode 100644 freqtrade/pairlist/LowPriceFilter.py create mode 100644 freqtrade/pairlist/PrecisionFilter.py create mode 100644 freqtrade/resolvers/pairlistfilter_resolver.py diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 5afb0c4c2..eb6af9d52 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -6,10 +6,11 @@ Provides lists as configured in config.json """ import logging from abc import ABC, abstractmethod -from typing import List +from typing import Dict, List from freqtrade.exchange import market_is_active - +from freqtrade.pairlist.IPairListFilter import IPairListFilter +from freqtrade.resolvers.pairlistfilter_resolver import PairListFilterResolver logger = logging.getLogger(__name__) @@ -21,6 +22,12 @@ class IPairList(ABC): self._config = config self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._filters = self._config.get('pairlist', {}).get('filters', {}) + self._pairlistfilters: List[IPairListFilter] = [] + for pl_filter in self._filters.keys(): + self._pairlistfilters.append( + PairListFilterResolver(pl_filter, freqtrade, self._config).pairlistfilter + ) @property def name(self) -> str: @@ -60,7 +67,23 @@ class IPairList(ABC): -> Please overwrite in subclasses """ - def _validate_whitelist(self, whitelist: List[str]) -> List[str]: + def validate_whitelist(self, pairlist: List[str], + tickers: List[Dict] = []) -> List[str]: + """ + Validate pairlist against active markets and blacklist. + Run PairlistFilters if these are configured. + """ + pairlist = self._whitelist_for_active_markets(pairlist) + + if not tickers: + # Refresh tickers if they are not used by the parent Pairlist + tickers = self._freqtrade.exchange.get_tickers() + + for pl_filter in self._pairlistfilters: + pairlist = pl_filter.filter_pairlist(pairlist, tickers) + return pairlist + + def _whitelist_for_active_markets(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary :param whitelist: the sorted list of pairs the user might want to trade @@ -69,7 +92,7 @@ class IPairList(ABC): """ markets = self._freqtrade.exchange.markets - sanitized_whitelist = set() + sanitized_whitelist: List[str] = [] for pair in whitelist: # pair is not in the generated dynamic market, or in the blacklist ... ignore it if (pair in self.blacklist or pair not in markets @@ -83,7 +106,8 @@ class IPairList(ABC): if not market_is_active(market): logger.info(f"Ignoring {pair} from whitelist. Market is not active.") continue - sanitized_whitelist.add(pair) + if pair not in sanitized_whitelist: + sanitized_whitelist.append(pair) # We need to remove pairs that are unknown - return list(sanitized_whitelist) + return sanitized_whitelist diff --git a/freqtrade/pairlist/IPairListFilter.py b/freqtrade/pairlist/IPairListFilter.py new file mode 100644 index 000000000..4b43f0e9f --- /dev/null +++ b/freqtrade/pairlist/IPairListFilter.py @@ -0,0 +1,18 @@ +import logging +from abc import ABC, abstractmethod +from typing import Dict, List + +logger = logging.getLogger(__name__) + + +class IPairListFilter(ABC): + + def __init__(self, freqtrade, config: dict) -> None: + self._freqtrade = freqtrade + self._config = config + + @abstractmethod + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + """ + Method doing the filtering + """ diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py new file mode 100644 index 000000000..499dd0c15 --- /dev/null +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -0,0 +1,48 @@ +import logging +from copy import deepcopy +from typing import Dict, List + +from freqtrade.pairlist.IPairListFilter import IPairListFilter + +logger = logging.getLogger(__name__) + + +class LowPriceFilter(IPairListFilter): + + def __init__(self, freqtrade, config: dict) -> None: + super().__init__(freqtrade, config) + + self._low_price_percent = config['pairlist']['filters']['LowPriceFilter'].get( + 'low_price_percent', 0) + + def _validate_precision_filter_lowprice(self, ticker) -> bool: + """ + Check if if one price-step is > than a certain barrier. + :param ticker: ticker dict as returned from ccxt.load_markets() + :param precision: Precision + :return: True if the pair can stay, false if it should be removed + """ + precision = self._freqtrade.exchange.markets[ticker['symbol']]['precision']['price'] + + compare = ticker['last'] + 1 / pow(10, precision) + changeperc = (compare - ticker['last']) / ticker['last'] + if changeperc > self._low_price_percent: + logger.info(f"Removed {ticker['symbol']} from whitelist, " + f"because 1 unit is {changeperc * 100:.3f}%") + return False + return True + + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + """ + Method doing the filtering + """ + + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + ticker = [t for t in tickers if t['symbol'] == p][0] + + # Filter out assets which would not allow setting a stoploss + if self._low_price_percent and not self._validate_precision_filter_lowprice(ticker): + pairlist.remove(p) + + return pairlist diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py new file mode 100644 index 000000000..c720b8e61 --- /dev/null +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -0,0 +1,51 @@ +import logging +from copy import deepcopy +from typing import Dict, List + +from freqtrade.pairlist.IPairListFilter import IPairListFilter + +logger = logging.getLogger(__name__) + + +class PrecisionFilter(IPairListFilter): + + def __init__(self, freqtrade, config: dict) -> None: + super().__init__(freqtrade, config) + + def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool: + """ + Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very + low value pairs. + :param ticker: ticker dict as returned from ccxt.load_markets() + :param stoploss: stoploss value as set in the configuration + (already cleaned to be 1 - stoploss) + :return: True if the pair can stay, false if it should be removed + """ + stop_price = self._freqtrade.get_target_bid(ticker["symbol"], ticker) * stoploss + # Adjust stop-prices to precision + sp = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], stop_price) + stop_gap_price = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], + stop_price * 0.99) + logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") + if sp <= stop_gap_price: + logger.info(f"Removed {ticker['symbol']} from whitelist, " + f"because stop price {sp} would be <= stop limit {stop_gap_price}") + return False + return True + + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + """ + Method doing the filtering + """ + if self._freqtrade.strategy.stoploss is not None: + # Precalculate sanitized stoploss value to avoid recalculation for every pair + stoploss = 1 - abs(self._freqtrade.strategy.stoploss) + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + ticker = [t for t in tickers if t['symbol'] == p][0] + # Filter out assets which would not allow setting a stoploss + if (stoploss and not self._validate_precision_filter(ticker, stoploss)): + pairlist.remove(p) + continue + + return pairlist diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 5896e814a..074652b25 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -27,4 +27,4 @@ class StaticPairList(IPairList): """ Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively """ - self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist']) + self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 7c17c6d57..911bb3bda 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -5,7 +5,6 @@ Provides lists as configured in config.json """ import logging -from copy import deepcopy from typing import List from cachetools import TTLCache, cached @@ -30,7 +29,6 @@ class VolumePairList(IPairList): self._number_pairs = self._whitelistconf['number_assets'] self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') self._precision_filter = self._whitelistconf.get('precision_filter', True) - self._low_price_percent_filter = self._whitelistconf.get('low_price_percent_filter', None) if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -60,44 +58,6 @@ class VolumePairList(IPairList): self._whitelist = self._gen_pair_whitelist( self._config['stake_currency'], self._sort_key) - def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool: - """ - Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very - low value pairs. - :param ticker: ticker dict as returned from ccxt.load_markets() - :param stoploss: stoploss value as set in the configuration - (already cleaned to be 1 - stoploss) - :return: True if the pair can stay, false if it should be removed - """ - stop_price = self._freqtrade.get_target_bid(ticker["symbol"], ticker) * stoploss - # Adjust stop-prices to precision - sp = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], stop_price) - stop_gap_price = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], - stop_price * 0.99) - logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") - if sp <= stop_gap_price: - logger.info(f"Removed {ticker['symbol']} from whitelist, " - f"because stop price {sp} would be <= stop limit {stop_gap_price}") - return False - return True - - def _validate_precision_filter_lowprice(self, ticker) -> bool: - """ - Check if if one price-step is > than a certain barrier. - :param ticker: ticker dict as returned from ccxt.load_markets() - :param precision: Precision - :return: True if the pair can stay, false if it should be removed - """ - precision = self._freqtrade.exchange.markets[ticker['symbol']]['precision']['price'] - - compare = ticker['last'] + 1 / pow(10, precision) - changeperc = (compare - ticker['last']) / ticker['last'] - if changeperc > self._low_price_percent_filter: - logger.info(f"Removed {ticker['symbol']} from whitelist, " - f"because 1 unit is {changeperc * 100:.3f}%") - return False - return True - @cached(TTLCache(maxsize=1, ttl=1800)) def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: """ @@ -114,26 +74,8 @@ class VolumePairList(IPairList): and v[key] is not None)] sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) # Validate whitelist to only have active market pairs - valid_pairs = self._validate_whitelist([s['symbol'] for s in sorted_tickers]) - valid_tickers = [t for t in sorted_tickers if t["symbol"] in valid_pairs] + pairs = self.validate_whitelist([s['symbol'] for s in sorted_tickers], tickers) - stoploss = None - if self._freqtrade.strategy.stoploss is not None: - # Precalculate sanitized stoploss value to avoid recalculation for every pair - stoploss = 1 - abs(self._freqtrade.strategy.stoploss) - - # Copy list since we're modifying this list - for t in deepcopy(valid_tickers): - # Filter out assets which would not allow setting a stoploss - if (stoploss and self._precision_filter - and not self._validate_precision_filter(t, stoploss)): - valid_tickers.remove(t) - continue - if self._low_price_percent_filter and not self._validate_precision_filter_lowprice(t,): - valid_tickers.remove(t) - continue - - pairs = [s['symbol'] for s in valid_tickers] - logger.info(f"Searching pairs: {pairs[:self._number_pairs]}") + logger.info(f"Searching {self._number_pairs} pairs: {pairs[:self._number_pairs]}") return pairs diff --git a/freqtrade/resolvers/pairlistfilter_resolver.py b/freqtrade/resolvers/pairlistfilter_resolver.py new file mode 100644 index 000000000..bf86d1c6c --- /dev/null +++ b/freqtrade/resolvers/pairlistfilter_resolver.py @@ -0,0 +1,53 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom pairlists +""" +import logging +from pathlib import Path + +from freqtrade import OperationalException +from freqtrade.pairlist.IPairListFilter import IPairListFilter +from freqtrade.resolvers import IResolver + +logger = logging.getLogger(__name__) + + +class PairListFilterResolver(IResolver): + """ + This class contains all the logic to load custom PairList class + """ + + __slots__ = ['pairlistfilter'] + + def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + self.pairlistfilter = self._load_pairlist(pairlist_name, config, + kwargs={'freqtrade': freqtrade, + 'config': config}) + + def _load_pairlist( + self, pairlistfilter_name: str, config: dict, kwargs: dict) -> IPairListFilter: + """ + Search and loads the specified pairlist. + :param pairlistfilter_name: name of the module to import + :param config: configuration dictionary + :param extra_dir: additional directory to search for the given pairlist + :return: PairList instance or None + """ + current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() + + abs_paths = self.build_search_paths(config, current_path=current_path, + user_subdir=None, extra_dir=None) + + pairlist = self._load_object(paths=abs_paths, object_type=IPairListFilter, + object_name=pairlistfilter_name, kwargs=kwargs) + if pairlist: + return pairlist + raise OperationalException( + f"Impossible to load PairlistFilter '{pairlistfilter_name}'. This class does not exist " + "or contains Python code errors." + )