Merge pull request #2954 from freqtrade/rate_caching

Improve and fix buy / sell Rate caching

freqtrade/exchange/exchange.py

@@ -66,8 +66,6 @@ class Exchange:
 
         self._config.update(config)
 
-        self._cached_ticker: Dict[str, Any] = {}
-
         # Holds last candle refreshed time of each pair
         self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {}
         # Timestamp of last markets refresh
@@ -591,28 +589,17 @@ class Exchange:
             raise OperationalException(e) from e
 
     @retrier
-    def fetch_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict:
+    def fetch_ticker(self, pair: str) -> dict:
-        if refresh or pair not in self._cached_ticker.keys():
+        try:
-            try:
+            if pair not in self._api.markets or not self._api.markets[pair].get('active'):
-                if pair not in self._api.markets or not self._api.markets[pair].get('active'):
+                raise DependencyException(f"Pair {pair} not available")
-                    raise DependencyException(f"Pair {pair} not available")
+            data = self._api.fetch_ticker(pair)
-                data = self._api.fetch_ticker(pair)
+            return data
-                try:
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-                    self._cached_ticker[pair] = {
+            raise TemporaryError(
-                        'bid': float(data['bid']),
+                f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e
-                        'ask': float(data['ask']),
+        except ccxt.BaseError as e:
-                    }
+            raise OperationalException(e) from e
-                except KeyError:
-                    logger.debug("Could not cache ticker data for %s", pair)
-                return data
-            except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-                raise TemporaryError(
-                    f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e
-            except ccxt.BaseError as e:
-                raise OperationalException(e) from e
-        else:
-            logger.info("returning cached ticker-data for %s", pair)
-            return self._cached_ticker[pair]
 
     def get_historic_ohlcv(self, pair: str, timeframe: str,
                            since_ms: int) -> List:

freqtrade/freqtradebot.py

@@ -10,6 +10,7 @@ from threading import Lock
 from typing import Any, Dict, List, Optional, Tuple
 
 import arrow
+from cachetools import TTLCache
 from requests.exceptions import RequestException
 
 from freqtrade import __version__, constants, persistence
@@ -51,6 +52,9 @@ class FreqtradeBot:
         # Init objects
         self.config = config
 
+        self._sell_rate_cache = TTLCache(maxsize=100, ttl=5)
+        self._buy_rate_cache = TTLCache(maxsize=100, ttl=5)
+
         self.strategy: IStrategy = StrategyResolver.load_strategy(self.config)
 
         # Check config consistency here since strategies can set certain options
@@ -224,11 +228,20 @@ class FreqtradeBot:
 
         return trades_created
 
-    def get_buy_rate(self, pair: str, refresh: bool, tick: Dict = None) -> float:
+    def get_buy_rate(self, pair: str, refresh: bool) -> float:
         """
         Calculates bid target between current ask price and last price
+        :param pair: Pair to get rate for
+        :param refresh: allow cached data
         :return: float: Price
         """
+        if not refresh:
+            rate = self._buy_rate_cache.get(pair)
+            # Check if cache has been invalidated
+            if rate:
+                logger.info(f"Using cached buy rate for {pair}.")
+                return rate
+
         config_bid_strategy = self.config.get('bid_strategy', {})
         if 'use_order_book' in config_bid_strategy and\
                 config_bid_strategy.get('use_order_book', False):
@@ -241,11 +254,8 @@ class FreqtradeBot:
             logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate)
             used_rate = order_book_rate
         else:
-            if not tick:
+            logger.info('Using Last Ask / Last Price')
-                logger.info('Using Last Ask / Last Price')
+            ticker = self.exchange.fetch_ticker(pair)
-                ticker = self.exchange.fetch_ticker(pair, refresh)
-            else:
-                ticker = tick
             if ticker['ask'] < ticker['last']:
                 ticker_rate = ticker['ask']
             else:
@@ -253,6 +263,8 @@ class FreqtradeBot:
                 ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask'])
             used_rate = ticker_rate
 
+        self._buy_rate_cache[pair] = used_rate
+
         return used_rate
 
     def get_trade_stake_amount(self, pair: str) -> float:
@@ -556,7 +568,7 @@ class FreqtradeBot:
         """
         Sends rpc notification when a buy cancel occured.
         """
-        current_rate = self.get_buy_rate(trade.pair, True)
+        current_rate = self.get_buy_rate(trade.pair, False)
 
         msg = {
             'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
@@ -611,8 +623,17 @@ class FreqtradeBot:
         The orderbook portion is only used for rpc messaging, which would otherwise fail
         for BitMex (has no bid/ask in fetch_ticker)
         or remain static in any other case since it's not updating.
+        :param pair: Pair to get rate for
+        :param refresh: allow cached data
         :return: Bid rate
         """
+        if not refresh:
+            rate = self._sell_rate_cache.get(pair)
+            # Check if cache has been invalidated
+            if rate:
+                logger.info(f"Using cached sell rate for {pair}.")
+                return rate
+
         config_ask_strategy = self.config.get('ask_strategy', {})
         if config_ask_strategy.get('use_order_book', False):
             logger.debug('Using order book to get sell rate')
@@ -621,7 +642,8 @@ class FreqtradeBot:
             rate = order_book['bids'][0][0]
 
         else:
-            rate = self.exchange.fetch_ticker(pair, refresh)['bid']
+            rate = self.exchange.fetch_ticker(pair)['bid']
+        self._sell_rate_cache[pair] = rate
         return rate
 
     def handle_trade(self, trade: Trade) -> bool:
@@ -1048,7 +1070,7 @@ class FreqtradeBot:
         """
         profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
         profit_trade = trade.calc_profit(rate=profit_rate)
-        current_rate = self.get_sell_rate(trade.pair, True)
+        current_rate = self.get_sell_rate(trade.pair, False)
         profit_percent = trade.calc_profit_ratio(profit_rate)
         gain = "profit" if profit_percent > 0 else "loss"
 

tests/exchange/test_exchange.py

@@ -1121,25 +1121,16 @@ def test_fetch_ticker(default_conf, mocker, exchange_name):
     assert ticker['bid'] == 0.5
     assert ticker['ask'] == 1
 
-    assert 'ETH/BTC' in exchange._cached_ticker
-    assert exchange._cached_ticker['ETH/BTC']['bid'] == 0.5
-    assert exchange._cached_ticker['ETH/BTC']['ask'] == 1
-
-    # Test caching
-    api_mock.fetch_ticker = MagicMock()
-    exchange.fetch_ticker(pair='ETH/BTC', refresh=False)
-    assert api_mock.fetch_ticker.call_count == 0
-
     ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
                            "fetch_ticker", "fetch_ticker",
-                           pair='ETH/BTC', refresh=True)
+                           pair='ETH/BTC')
 
     api_mock.fetch_ticker = MagicMock(return_value={})
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-    exchange.fetch_ticker(pair='ETH/BTC', refresh=True)
+    exchange.fetch_ticker(pair='ETH/BTC')
 
     with pytest.raises(DependencyException, match=r'Pair XRP/ETH not available'):
-        exchange.fetch_ticker(pair='XRP/ETH', refresh=True)
+        exchange.fetch_ticker(pair='XRP/ETH')
 
 
 @pytest.mark.parametrize("exchange_name", EXCHANGES)

tests/rpc/test_rpc.py

@@ -65,10 +65,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'open_order': '(limit buy rem=0.00000000)'
     } == results[0]
 
-    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker',
+    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate',
                  MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available")))
-    # invalidate ticker cache
-    rpc._freqtrade.exchange._cached_ticker = {}
     results = rpc._rpc_trade_status()
     assert isnan(results[0]['current_profit'])
     assert isnan(results[0]['current_rate'])
@@ -134,10 +132,8 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None:
     assert 'ETH/BTC' in result[0][1]
     assert '-0.59% (-0.09)' == result[0][3]
 
-    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker',
+    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate',
                  MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available")))
-    # invalidate ticker cache
-    rpc._freqtrade.exchange._cached_ticker = {}
     result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD')
     assert 'instantly' == result[0][2]
     assert 'ETH/BTC' in result[0][1]
@@ -260,10 +256,8 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee,
     assert prec_satoshi(stats['best_rate'], 6.2)
 
     # Test non-available pair
-    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker',
+    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate',
                  MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available")))
-    # invalidate ticker cache
-    rpc._freqtrade.exchange._cached_ticker = {}
     stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency)
     assert stats['trade_count'] == 2
     assert stats['first_trade_date'] == 'just now'

tests/test_freqtradebot.py

@@ -915,13 +915,21 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None:
     (5, 10, 1.0, 5),  # last bigger than ask
     (5, 10, 0.5, 5),  # last bigger than ask
 ])
-def test_get_buy_rate(mocker, default_conf, ask, last, last_ab, expected) -> None:
+def test_get_buy_rate(mocker, default_conf, caplog, ask, last, last_ab, expected) -> None:
     default_conf['bid_strategy']['ask_last_balance'] = last_ab
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     mocker.patch('freqtrade.exchange.Exchange.fetch_ticker',
                  MagicMock(return_value={'ask': ask, 'last': last}))
 
     assert freqtrade.get_buy_rate('ETH/BTC', True) == expected
+    assert not log_has("Using cached buy rate for ETH/BTC.", caplog)
+
+    assert freqtrade.get_buy_rate('ETH/BTC', False) == expected
+    assert log_has("Using cached buy rate for ETH/BTC.", caplog)
+    # Running a 2nd time with Refresh on!
+    caplog.clear()
+    assert freqtrade.get_buy_rate('ETH/BTC', True) == expected
+    assert not log_has("Using cached buy rate for ETH/BTC.", caplog)
 
 
 def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None:
@@ -3614,7 +3622,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order
     assert freqtrade.handle_trade(trade) is True
 
 
-def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None:
+def test_get_sell_rate(default_conf, mocker, caplog, ticker, order_book_l2) -> None:
 
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
@@ -3626,8 +3634,15 @@ def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None:
     # Test regular mode
     ft = get_patched_freqtradebot(mocker, default_conf)
     rate = ft.get_sell_rate(pair, True)
+    assert not log_has("Using cached sell rate for ETH/BTC.", caplog)
     assert isinstance(rate, float)
     assert rate == 0.00001098
+    # Use caching
+    rate = ft.get_sell_rate(pair, False)
+    assert rate == 0.00001098
+    assert log_has("Using cached sell rate for ETH/BTC.", caplog)
+
+    caplog.clear()
 
     # Test orderbook mode
     default_conf['ask_strategy']['use_order_book'] = True
@@ -3635,8 +3650,12 @@ def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None:
     default_conf['ask_strategy']['order_book_max'] = 2
     ft = get_patched_freqtradebot(mocker, default_conf)
     rate = ft.get_sell_rate(pair, True)
+    assert not log_has("Using cached sell rate for ETH/BTC.", caplog)
     assert isinstance(rate, float)
     assert rate == 0.043936
+    rate = ft.get_sell_rate(pair, False)
+    assert rate == 0.043936
+    assert log_has("Using cached sell rate for ETH/BTC.", caplog)
 
 
 def test_startup_state(default_conf, mocker):