From b7afcf3416aaca550edce9485d01b783a8cb1d8a Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sat, 16 Feb 2019 22:56:04 +0100 Subject: [PATCH 01/11] add VolumePrecisionPairList --- freqtrade/__init__.py | 2 +- freqtrade/constants.py | 2 +- freqtrade/data/history.py | 6 +- freqtrade/pairlist/VolumePrecisionPairList.py | 86 +++++++++++++++++++ 4 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 freqtrade/pairlist/VolumePrecisionPairList.py diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index ca148f518..0d1ae9c26 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.18.1-dev' +__version__ = '0.18.2-dev' class DependencyException(BaseException): diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 8fbcdfed7..840d348f0 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -17,7 +17,7 @@ REQUIRED_ORDERTIF = ['buy', 'sell'] REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList'] +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'VolumePrecisionPairList'] TICKER_INTERVAL_MINUTES = { '1m': 1, diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 7d89f7ad6..a7a3a61cf 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -229,7 +229,7 @@ def download_pair_history(datadir: Optional[Path], misc.file_dump_json(filename, data) return True - except BaseException: - logger.info('Failed to download the pair: "%s", Interval: %s', - pair, tick_interval) + except BaseException as e: + logger.info('Failed to download the pair: "%s", Interval: %s\n' + 'Error message: %s', pair, tick_interval, e) return False diff --git a/freqtrade/pairlist/VolumePrecisionPairList.py b/freqtrade/pairlist/VolumePrecisionPairList.py new file mode 100644 index 000000000..634988668 --- /dev/null +++ b/freqtrade/pairlist/VolumePrecisionPairList.py @@ -0,0 +1,86 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from typing import List +from cachetools import TTLCache, cached + +from freqtrade.pairlist.IPairList import IPairList +from freqtrade import OperationalException +logger = logging.getLogger(__name__) + +SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] + + +class VolumePrecisionPairList(IPairList): + + def __init__(self, freqtrade, config: dict) -> None: + super().__init__(freqtrade, config) + self._whitelistconf = self._config.get('pairlist', {}).get('config') + if 'number_assets' not in self._whitelistconf: + raise OperationalException( + f'`number_assets` not specified. Please check your configuration ' + 'for "pairlist.config.number_assets"') + self._number_pairs = self._whitelistconf['number_assets'] + self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') + + if not self._freqtrade.exchange.exchange_has('fetchTickers'): + raise OperationalException( + 'Exchange does not support dynamic whitelist.' + 'Please edit your config and restart the bot' + ) + if not self._validate_keys(self._sort_key): + raise OperationalException( + f'key {self._sort_key} not in {SORT_VALUES}') + + def _validate_keys(self, key): + return key in SORT_VALUES + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + -> Please overwrite in subclasses + """ + return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." + + def refresh_pairlist(self) -> None: + """ + Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + -> Please overwrite in subclasses + """ + # Generate dynamic whitelist + pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) + # Validate whitelist to only have active market pairs + self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] + + @cached(TTLCache(maxsize=1, ttl=1800)) + def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: + """ + Updates the whitelist with with a dynamically generated list + :param base_currency: base currency as str + :param key: sort key (defaults to 'quoteVolume') + :return: List of pairs + """ + + tickers = self._freqtrade.exchange.get_tickers() + # check length so that we make sure that '/' is actually in the string + tickers = [v for k, v in tickers.items() + if len(k.split('/')) == 2 and k.split('/')[1] == base_currency] + + if self._freqtrade.strategy.stoploss is not None: + precisions = [self._freqtrade.exchange.markets[ + t["symbol"]]["precision"].get("price") for t in tickers] + tickers = [t for t, p in zip(tickers, precisions) if ( + self._freqtrade.exchange.symbol_price_prec( + t["symbol"], + self._freqtrade.get_target_bid( + t["symbol"], t) * (1 + self._freqtrade.strategy.stoploss) + ) < self._freqtrade.get_target_bid(t["symbol"], t) + )] + + sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) + pairs = [s['symbol'] for s in sorted_tickers] + return pairs From 0572336ff790b39bc5d3719504590cc471b6251d Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sun, 17 Feb 2019 16:12:40 +0100 Subject: [PATCH 02/11] revert changes to history --- freqtrade/data/history.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index a7a3a61cf..7d89f7ad6 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -229,7 +229,7 @@ def download_pair_history(datadir: Optional[Path], misc.file_dump_json(filename, data) return True - except BaseException as e: - logger.info('Failed to download the pair: "%s", Interval: %s\n' - 'Error message: %s', pair, tick_interval, e) + except BaseException: + logger.info('Failed to download the pair: "%s", Interval: %s', + pair, tick_interval) return False From cc0fae8e4e71a5ca2f57d0cc2c1766a867fcbb5b Mon Sep 17 00:00:00 2001 From: iuvbio Date: Fri, 22 Feb 2019 21:13:08 +0100 Subject: [PATCH 03/11] change < to <= --- freqtrade/pairlist/VolumePrecisionPairList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/pairlist/VolumePrecisionPairList.py b/freqtrade/pairlist/VolumePrecisionPairList.py index 634988668..e7147c3b2 100644 --- a/freqtrade/pairlist/VolumePrecisionPairList.py +++ b/freqtrade/pairlist/VolumePrecisionPairList.py @@ -78,7 +78,7 @@ class VolumePrecisionPairList(IPairList): t["symbol"], self._freqtrade.get_target_bid( t["symbol"], t) * (1 + self._freqtrade.strategy.stoploss) - ) < self._freqtrade.get_target_bid(t["symbol"], t) + ) <= self._freqtrade.get_target_bid(t["symbol"], t) )] sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) From 98bca30dfb19825c2e47899d48065465c4a338d8 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Fri, 22 Feb 2019 21:16:31 +0100 Subject: [PATCH 04/11] reorganize imports --- freqtrade/pairlist/VolumePrecisionPairList.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/pairlist/VolumePrecisionPairList.py b/freqtrade/pairlist/VolumePrecisionPairList.py index e7147c3b2..8caaa6f39 100644 --- a/freqtrade/pairlist/VolumePrecisionPairList.py +++ b/freqtrade/pairlist/VolumePrecisionPairList.py @@ -3,13 +3,16 @@ Static List provider Provides lists as configured in config.json - """ +""" import logging from typing import List + from cachetools import TTLCache, cached from freqtrade.pairlist.IPairList import IPairList from freqtrade import OperationalException + + logger = logging.getLogger(__name__) SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] From 24c587518ad60775f5fa3b69ba52c438034b42b6 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sat, 2 Mar 2019 17:24:28 +0100 Subject: [PATCH 05/11] add precision_filter --- config_full.json.example | 3 ++- freqtrade/pairlist/IPairList.py | 3 ++- freqtrade/pairlist/VolumePairList.py | 32 ++++++++++++++++++++++------ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 23a36dd4c..0f46a62e3 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -49,7 +49,8 @@ "method": "VolumePairList", "config": { "number_assets": 20, - "sort_key": "quoteVolume" + "sort_key": "quoteVolume", + "precision_filter": false } }, "exchange": { diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 6b5b0db4b..4e9e28b3d 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -18,6 +18,7 @@ class IPairList(ABC): self._config = config self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._markets = self._freqtrade.exchange.markets @property def name(self) -> str: @@ -66,7 +67,7 @@ class IPairList(ABC): black_listed """ sanitized_whitelist = whitelist - markets = self._freqtrade.exchange.get_markets() + markets = list(self._markets.values()) # Filter to markets in stake currency markets = [m for m in markets if m['quote'] == self._config['stake_currency']] diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 262e4bf59..7f1985d43 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -1,5 +1,5 @@ """ -Static List provider +Volume PairList provider Provides lists as configured in config.json @@ -26,6 +26,7 @@ class VolumePairList(IPairList): 'for "pairlist.config.number_assets"') self._number_pairs = self._whitelistconf['number_assets'] self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') + self._precision_filter = self._whitelistconf.get('precision_filter', False) if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -52,9 +53,9 @@ class VolumePairList(IPairList): -> Please overwrite in subclasses """ # Generate dynamic whitelist - pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) - # Validate whitelist to only have active market pairs - self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] + self._whitelist = self._gen_pair_whitelist( + self._config['stake_currency'], self._sort_key)[:self._number_pairs] + logger.info(f"Searching pairs: {self._whitelist}") @cached(TTLCache(maxsize=1, ttl=1800)) def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: @@ -69,7 +70,26 @@ class VolumePairList(IPairList): # check length so that we make sure that '/' is actually in the string tickers = [v for k, v in tickers.items() if len(k.split('/')) == 2 and k.split('/')[1] == base_currency] - sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) - pairs = [s['symbol'] for s in sorted_tickers] + # Validate whitelist to only have active market pairs + valid_pairs = self._validate_whitelist([s['symbol'] for s in sorted_tickers]) + valid_tickers = [t for t in sorted_tickers if t["symbol"] in valid_pairs] + + if self._freqtrade.strategy.stoploss is not None and self._precision_filter: + + logger.debug(f"Markets: {list(self._markets)}") + stop_prices = [self._freqtrade.get_target_bid(t["symbol"], t) + * (1 + self._freqtrade.strategy.stoploss) for t in valid_tickers] + rates = [sp * 0.99 for sp in stop_prices] + logger.debug("\n".join([f"{sp} : {r}" for sp, r in zip(stop_prices[:10], rates[:10])])) + for i, t in enumerate(valid_tickers): + sp = self._freqtrade.exchange.symbol_price_prec(t["symbol"], stop_prices[i]) + r = self._freqtrade.exchange.symbol_price_prec(t["symbol"], rates[i]) + logger.debug(f"{t['symbol']} - {sp} : {r}") + if sp <= r: + logger.info(f"Removed {t['symbol']} from whitelist, " + f"because stop price {sp} would be <= stop limit {r}") + valid_tickers.remove(t) + + pairs = [s['symbol'] for s in valid_tickers] return pairs From c36fa0c7e2b368a2b20ba1997de8394e6aec4f39 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sat, 2 Mar 2019 17:24:48 +0100 Subject: [PATCH 06/11] add ticker argumet to get_target_bid --- freqtrade/freqtradebot.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 92bdbc042..86f739f2f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -206,7 +206,7 @@ class FreqtradeBot(object): self.state = State.STOPPED return state_changed - def get_target_bid(self, pair: str) -> float: + def get_target_bid(self, pair: str, ticker: Dict = None) -> float: """ Calculates bid target between current ask price and last price :return: float: Price @@ -223,8 +223,9 @@ class FreqtradeBot(object): logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) used_rate = order_book_rate else: - logger.info('Using Last Ask / Last Price') - ticker = self.exchange.get_ticker(pair) + if not ticker: + logger.info('Using Last Ask / Last Price') + ticker = self.exchange.get_ticker(pair) if ticker['ask'] < ticker['last']: ticker_rate = ticker['ask'] else: @@ -269,12 +270,10 @@ class FreqtradeBot(object): return stake_amount def _get_min_pair_stake_amount(self, pair: str, price: float) -> Optional[float]: - markets = self.exchange.get_markets() - markets = [m for m in markets if m['symbol'] == pair] - if not markets: - raise ValueError(f'Can\'t get market information for symbol {pair}') - - market = markets[0] + try: + market = self.exchange.markets[pair] + except KeyError: + raise ValueError(f"Can't get market information for symbol {pair}") if 'limits' not in market: return None From e1ae0d7e903df034a17c4f1608420126cc41042a Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sat, 2 Mar 2019 18:53:42 +0100 Subject: [PATCH 07/11] remove markets changes --- freqtrade/exchange/exchange.py | 4 +- freqtrade/freqtradebot.py | 16 ++-- freqtrade/pairlist/IPairList.py | 6 +- freqtrade/tests/conftest.py | 94 +++++++++++++++++++++++ freqtrade/tests/exchange/test_exchange.py | 2 +- freqtrade/tests/pairlist/test_pairlist.py | 10 ++- 6 files changed, 117 insertions(+), 15 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 25d3e62c0..0ca939ec5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -715,7 +715,9 @@ class Exchange(object): @retrier def get_markets(self) -> List[dict]: try: - return self._api.fetch_markets() + markets = self._api.fetch_markets() + self.markets.update({m["symbol"]: m for m in markets}) + return markets except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not load markets due to {e.__class__.__name__}. Message: {e}') diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 86f739f2f..28a7c9146 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -206,7 +206,7 @@ class FreqtradeBot(object): self.state = State.STOPPED return state_changed - def get_target_bid(self, pair: str, ticker: Dict = None) -> float: + def get_target_bid(self, pair: str, tick: Dict = None) -> float: """ Calculates bid target between current ask price and last price :return: float: Price @@ -223,9 +223,11 @@ class FreqtradeBot(object): logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) used_rate = order_book_rate else: - if not ticker: + if not tick: logger.info('Using Last Ask / Last Price') ticker = self.exchange.get_ticker(pair) + else: + ticker = tick if ticker['ask'] < ticker['last']: ticker_rate = ticker['ask'] else: @@ -270,10 +272,12 @@ class FreqtradeBot(object): return stake_amount def _get_min_pair_stake_amount(self, pair: str, price: float) -> Optional[float]: - try: - market = self.exchange.markets[pair] - except KeyError: - raise ValueError(f"Can't get market information for symbol {pair}") + markets = self.exchange.get_markets() + markets = [m for m in markets if m['symbol'] == pair] + if not markets: + raise ValueError(f'Can\'t get market information for symbol {pair}') + + market = markets[0] if 'limits' not in market: return None diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 4e9e28b3d..7675c1eee 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -18,7 +18,7 @@ class IPairList(ABC): self._config = config self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self._markets = self._freqtrade.exchange.markets + self._markets = self._freqtrade.exchange.get_markets() @property def name(self) -> str: @@ -67,7 +67,7 @@ class IPairList(ABC): black_listed """ sanitized_whitelist = whitelist - markets = list(self._markets.values()) + markets = self._freqtrade.exchange.get_markets() # Filter to markets in stake currency markets = [m for m in markets if m['quote'] == self._config['stake_currency']] @@ -75,7 +75,7 @@ class IPairList(ABC): for market in markets: pair = market['symbol'] - # pair is not int the generated dynamic market, or in the blacklist ... ignore it + # pair is not in the generated dynamic market, or in the blacklist ... ignore it if pair not in whitelist or pair in self.blacklist: continue # else the pair is valid diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d6628d925..98171844a 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -375,6 +375,78 @@ def markets(): }, }, 'info': '', + }, + { + 'id': 'BTTBTC', + 'symbol': 'BTT/BTC', + 'base': 'BTT', + 'quote': 'BTC', + 'active': True, + 'precision': { + 'base': 8, + 'quote': 8, + 'amount': 0, + 'price': 8 + }, + 'limits': { + 'amount': { + 'min': 1.0, + 'max': 90000000.0 + }, + 'price': { + 'min': None, + 'max': None + }, + 'cost': { + 'min': 0.001, + 'max': None + } + }, + 'info': "", + }, + { + 'id': 'USDT-ETH', + 'symbol': 'ETH/USDT', + 'base': 'ETH', + 'quote': 'USDT', + 'precision': { + 'amount': 8, + 'price': 8 + }, + 'limits': { + 'amount': { + 'min': 0.02214286, + 'max': None + }, + 'price': { + 'min': 1e-08, + 'max': None + } + }, + 'active': True, + 'info': "" + }, + { + 'id': 'USDT-LTC', + 'symbol': 'LTC/USDT', + 'base': 'LTC', + 'quote': 'USDT', + 'active': True, + 'precision': { + 'amount': 8, + 'price': 8 + }, + 'limits': { + 'amount': { + 'min': 0.06646786, + 'max': None + }, + 'price': { + 'min': 1e-08, + 'max': None + } + }, + 'info': "" } ]) @@ -642,6 +714,28 @@ def tickers(): 'quoteVolume': 1401.65697943, 'info': {} }, + 'BTT/BTC': { + 'symbol': 'BTT/BTC', + 'timestamp': 1550936557206, + 'datetime': '2019-02-23T15:42:37.206Z', + 'high': 0.00000026, + 'low': 0.00000024, + 'bid': 0.00000024, + 'bidVolume': 2446894197.0, + 'ask': 0.00000025, + 'askVolume': 2447913837.0, + 'vwap': 0.00000025, + 'open': 0.00000026, + 'close': 0.00000024, + 'last': 0.00000024, + 'previousClose': 0.00000026, + 'change': -0.00000002, + 'percentage': -7.692, + 'average': None, + 'baseVolume': 4886464537.0, + 'quoteVolume': 1215.14489611, + 'info': {} + }, 'ETH/USDT': { 'symbol': 'ETH/USDT', 'timestamp': 1522014804118, diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 15da5e924..4fadfd4b7 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1264,7 +1264,7 @@ def test_get_markets(default_conf, mocker, markets): exchange = get_patched_exchange(mocker, default_conf, api_mock) ret = exchange.get_markets() assert isinstance(ret, list) - assert len(ret) == 6 + assert len(ret) == 9 assert ret[0]["id"] == "ethbtc" assert ret[0]["symbol"] == "ETH/BTC" diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index 9f90aac6e..e78404587 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -80,7 +80,7 @@ def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC'] + whitelist = ['ETH/BTC', 'TKN/BTC', 'BTT/BTC'] freqtradebot.pairlists.refresh_pairlist() assert whitelist == freqtradebot.pairlists.whitelist @@ -116,17 +116,19 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) # Test to retrieved BTC sorted on quoteVolume (default) whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume') - assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC'] + assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BTT/BTC'] # Test to retrieve BTC sorted on bidVolume whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='bidVolume') - assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC'] + assert whitelist == ['BTT/BTC', 'TKN/BTC', 'ETH/BTC'] # Test with USDT sorted on quoteVolume (default) + freqtrade.config['stake_currency'] = 'USDT' # this has to be set, otherwise markets are removed whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='USDT', key='quoteVolume') - assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT'] + assert whitelist == ['ETH/USDT', 'LTC/USDT'] # Test with ETH (our fixture does not have ETH, so result should be empty) + freqtrade.config['stake_currency'] = 'ETH' whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='ETH', key='quoteVolume') assert whitelist == [] From 064f6629ab9303faf6a921ec4d5fea4a44c49f4e Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sun, 3 Mar 2019 00:35:25 +0100 Subject: [PATCH 08/11] delete separate pairlist --- freqtrade/pairlist/VolumePrecisionPairList.py | 89 ------------------- 1 file changed, 89 deletions(-) delete mode 100644 freqtrade/pairlist/VolumePrecisionPairList.py diff --git a/freqtrade/pairlist/VolumePrecisionPairList.py b/freqtrade/pairlist/VolumePrecisionPairList.py deleted file mode 100644 index 8caaa6f39..000000000 --- a/freqtrade/pairlist/VolumePrecisionPairList.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -Static List provider - -Provides lists as configured in config.json - -""" -import logging -from typing import List - -from cachetools import TTLCache, cached - -from freqtrade.pairlist.IPairList import IPairList -from freqtrade import OperationalException - - -logger = logging.getLogger(__name__) - -SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] - - -class VolumePrecisionPairList(IPairList): - - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) - self._whitelistconf = self._config.get('pairlist', {}).get('config') - if 'number_assets' not in self._whitelistconf: - raise OperationalException( - f'`number_assets` not specified. Please check your configuration ' - 'for "pairlist.config.number_assets"') - self._number_pairs = self._whitelistconf['number_assets'] - self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') - - if not self._freqtrade.exchange.exchange_has('fetchTickers'): - raise OperationalException( - 'Exchange does not support dynamic whitelist.' - 'Please edit your config and restart the bot' - ) - if not self._validate_keys(self._sort_key): - raise OperationalException( - f'key {self._sort_key} not in {SORT_VALUES}') - - def _validate_keys(self, key): - return key in SORT_VALUES - - def short_desc(self) -> str: - """ - Short whitelist method description - used for startup-messages - -> Please overwrite in subclasses - """ - return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." - - def refresh_pairlist(self) -> None: - """ - Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively - -> Please overwrite in subclasses - """ - # Generate dynamic whitelist - pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) - # Validate whitelist to only have active market pairs - self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] - - @cached(TTLCache(maxsize=1, ttl=1800)) - def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: - """ - Updates the whitelist with with a dynamically generated list - :param base_currency: base currency as str - :param key: sort key (defaults to 'quoteVolume') - :return: List of pairs - """ - - tickers = self._freqtrade.exchange.get_tickers() - # check length so that we make sure that '/' is actually in the string - tickers = [v for k, v in tickers.items() - if len(k.split('/')) == 2 and k.split('/')[1] == base_currency] - - if self._freqtrade.strategy.stoploss is not None: - precisions = [self._freqtrade.exchange.markets[ - t["symbol"]]["precision"].get("price") for t in tickers] - tickers = [t for t, p in zip(tickers, precisions) if ( - self._freqtrade.exchange.symbol_price_prec( - t["symbol"], - self._freqtrade.get_target_bid( - t["symbol"], t) * (1 + self._freqtrade.strategy.stoploss) - ) <= self._freqtrade.get_target_bid(t["symbol"], t) - )] - - sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) - pairs = [s['symbol'] for s in sorted_tickers] - return pairs From df79098adc008922344efaf02e008cef211cf218 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sun, 3 Mar 2019 13:37:54 +0100 Subject: [PATCH 09/11] update docs --- docs/configuration.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 108e264c6..c4b26eba9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -96,7 +96,7 @@ To allow the bot to trade all the available `stake_currency` in your account set "stake_amount" : "unlimited", ``` -In this case a trade amount is calclulated as: +In this case a trade amount is calclulated as: ```python currency_balanse / (max_open_trades - current_open_trades) @@ -280,13 +280,15 @@ By default, a Static Pairlist is used (configured as `"pair_whitelist"` under th * `"VolumePairList"` * Formerly available as `--dynamic-whitelist []` * Selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume`, defaults to `quoteVolume`. + * Possibility to filter low-value coins that would not allow setting a stop loss ```json "pairlist": { "method": "VolumePairList", "config": { "number_assets": 20, - "sort_key": "quoteVolume" + "sort_key": "quoteVolume", + "precision_filter": false } }, ``` From e2cbb7e7dadba1d91390dc4c14c07e32fc5c5b23 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sun, 3 Mar 2019 13:41:51 +0100 Subject: [PATCH 10/11] remove remnants markets and precisionlist --- freqtrade/constants.py | 2 +- freqtrade/exchange/exchange.py | 4 +--- freqtrade/pairlist/IPairList.py | 1 - freqtrade/pairlist/VolumePairList.py | 3 +-- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d0fafbd26..4d0907d78 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -19,7 +19,7 @@ REQUIRED_ORDERTIF = ['buy', 'sell'] REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'VolumePrecisionPairList'] +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList'] TICKER_INTERVAL_MINUTES = { '1m': 1, diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index f5503002b..874ed93aa 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -657,9 +657,7 @@ class Exchange(object): @retrier def get_markets(self) -> List[dict]: try: - markets = self._api.fetch_markets() - self.markets.update({m["symbol"]: m for m in markets}) - return markets + return self._api.fetch_markets() except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not load markets due to {e.__class__.__name__}. Message: {e}') diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 7675c1eee..948abe113 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -18,7 +18,6 @@ class IPairList(ABC): self._config = config self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self._markets = self._freqtrade.exchange.get_markets() @property def name(self) -> str: diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 7f1985d43..eb03236e5 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -77,9 +77,8 @@ class VolumePairList(IPairList): if self._freqtrade.strategy.stoploss is not None and self._precision_filter: - logger.debug(f"Markets: {list(self._markets)}") stop_prices = [self._freqtrade.get_target_bid(t["symbol"], t) - * (1 + self._freqtrade.strategy.stoploss) for t in valid_tickers] + * (1 - abs(self._freqtrade.strategy.stoploss)) for t in valid_tickers] rates = [sp * 0.99 for sp in stop_prices] logger.debug("\n".join([f"{sp} : {r}" for sp, r in zip(stop_prices[:10], rates[:10])])) for i, t in enumerate(valid_tickers): From 3c5deb9aaf9882a7cd7ef2b0d30c0f77df4442ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Mar 2019 15:31:48 +0100 Subject: [PATCH 11/11] Add test for precision_remove ... BTT should not be in the list when that is enabled. --- freqtrade/tests/conftest.py | 1 + freqtrade/tests/pairlist/test_pairlist.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 98171844a..6237a27c9 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -667,6 +667,7 @@ def tickers(): 'vwap': 0.01869197, 'open': 0.018585, 'close': 0.018573, + 'last': 0.018799, 'baseVolume': 81058.66, 'quoteVolume': 2247.48374509, }, diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index e78404587..dd6ebb62c 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -113,6 +113,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, p, r: round(r, 8)) # Test to retrieved BTC sorted on quoteVolume (default) whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume') @@ -132,6 +133,15 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='ETH', key='quoteVolume') assert whitelist == [] + freqtrade.pairlists._precision_filter = True + freqtrade.config['stake_currency'] = 'BTC' + # Retest First 2 test-cases to make sure BTT is not in it (too low priced) + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume') + assert whitelist == ['ETH/BTC', 'TKN/BTC'] + + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='bidVolume') + assert whitelist == ['TKN/BTC', 'ETH/BTC'] + def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: default_conf['pairlist'] = {'method': 'VolumePairList',