Merge pull request #1915 from freqtrade/feat/drop_incomplete_optional

Make dropping the last candle optional (configured per exchange)

docs/configuration.md

@@ -303,6 +303,25 @@ This configuration enables binance, as well as rate limiting to avoid bans from
     Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings.
     We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step.
 
+#### Advanced FreqTrade Exchange configuration
+
+Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behaviours.
+
+Available options are listed in the exchange-class as `_ft_has_default`.
+
+For example, to test the order type `FOK` with Kraken, and modify candle_limit to 200 (so you only get 200 candles per call):
+
+```json
+"exchange": {
+    "name": "kraken",
+    "_ft_has_params": {
+        "order_time_in_force": ["gtc", "fok"],
+        "ohlcv_candle_limit": 200
+        }
+```
+
+!!! Warning
+    Please make sure to fully understand the impacts of these settings before modifying them.
 
 ### What values can be used for fiat_display_currency?
 

docs/developer.md

@@ -81,6 +81,51 @@ Please also run `self._validate_whitelist(pairs)` and to check and remove pairs
 This is a simple method used by `VolumePairList` - however serves as a good example.
 It implements caching (`@cached(TTLCache(maxsize=1, ttl=1800))`) as well as a configuration option to allow different (but similar) strategies to work with the same PairListProvider.
 
+## Implement a new Exchange (WIP)
+
+!!! Note
+    This section is a Work in Progress and is not a complete guide on how to test a new exchange with FreqTrade.
+
+Most exchanges supported by CCXT should work out of the box.
+
+### Stoploss On Exchange
+
+Check if the new exchange supports Stoploss on Exchange orders through their API.
+
+Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselfs. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects.
+
+### Incomplete candles
+
+While fetching OHLCV data, we're may end up getting incomplete candles (Depending on the exchange).
+To demonstrate this, we'll use daily candles (`"1d"`) to keep things simple.
+We query the api (`ct.fetch_ohlcv()`) for the timeframe and look at the date of the last entry. If this entry changes or shows the date of a "incomplete" candle, then we should drop this since having incomplete candles is problematic because indicators assume that only complete candles are passed to them, and will generate a lot of false buy signals. By default, we're therefore removing the last candle assuming it's incomplete.
+
+To check how the new exchange behaves, you can use the following snippet:
+
+``` python
+import ccxt
+from datetime import datetime
+from freqtrade.data.converter import parse_ticker_dataframe
+ct = ccxt.binance()
+timeframe = "1d"
+pair = "XLM/BTC"  # Make sure to use a pair that exists on that exchange!
+raw = ct.fetch_ohlcv(pair, timeframe=timeframe)
+
+# convert to dataframe
+df1 = parse_ticker_dataframe(raw, timeframe, drop_incomplete=False)
+
+print(df1["date"].tail(1))
+print(datetime.utcnow())
+```
+
+``` output
+19   2019-06-08 00:00:00+00:00
+2019-06-09 12:30:27.873327
+```
+
+The output will show the last entry from the Exchange as well as the current UTC date.
+If the day shows the same day, then the last candle can be assumed as incomplete and should be dropped (leave the setting `"ohlcv_partial_candle"` from the exchange-class untouched / True). Otherwise, set `"ohlcv_partial_candle"` to `False` to not drop Candles (shown in the example above).
+
 ## Creating a release
 
 This part of the documentation is aimed at maintainers, and shows how to create a release.

freqtrade/data/converter.py

@@ -10,14 +10,16 @@ from pandas import DataFrame, to_datetime
 logger = logging.getLogger(__name__)
 
 
-def parse_ticker_dataframe(ticker: list, ticker_interval: str,
-                           fill_missing: bool = True) -> DataFrame:
+def parse_ticker_dataframe(ticker: list, ticker_interval: str, *,
+                           fill_missing: bool = True,
+                           drop_incomplete: bool = True) -> DataFrame:
     """
     Converts a ticker-list (format ccxt.fetch_ohlcv) to a Dataframe
     :param ticker: ticker list, as returned by exchange.async_get_candle_history
     :param ticker_interval: ticker_interval (e.g. 5m). Used to fill up eventual missing data
     :param fill_missing: fill up missing candles with 0 candles
                          (see ohlcv_fill_up_missing_data for details)
+    :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete
     :return: DataFrame
     """
     logger.debug("Parsing tickerlist to dataframe")
@@ -43,7 +45,9 @@ def parse_ticker_dataframe(ticker: list, ticker_interval: str,
         'close': 'last',
         'volume': 'max',
     })
-    frame.drop(frame.tail(1).index, inplace=True)     # eliminate partial candle
+    # eliminate partial candle
+    if drop_incomplete:
+        frame.drop(frame.tail(1).index, inplace=True)
         logger.debug('Dropping last candle')
 
     if fill_missing:

freqtrade/data/history.py

@@ -81,10 +81,20 @@ def load_pair_history(pair: str,
                       timerange: TimeRange = TimeRange(None, None, 0, 0),
                       refresh_pairs: bool = False,
                       exchange: Optional[Exchange] = None,
-                      fill_up_missing: bool = True
+                      fill_up_missing: bool = True,
+                      drop_incomplete: bool = True
                       ) -> DataFrame:
     """
     Loads cached ticker history for the given pair.
+    :param pair: Pair to load data for
+    :param ticker_interval: Ticker-interval (e.g. "5m")
+    :param datadir: Path to the data storage location.
+    :param timerange: Limit data to be loaded to this timerange
+    :param refresh_pairs: Refresh pairs from exchange.
+        (Note: Requires exchange to be passed as well.)
+    :param exchange: Exchange object (needed when using "refresh_pairs")
+    :param fill_up_missing: Fill missing values with "No action"-candles
+    :param drop_incomplete: Drop last candle assuming it may be incomplete.
     :return: DataFrame with ohlcv data
     """
 
@@ -106,7 +116,9 @@ def load_pair_history(pair: str,
             logger.warning('Missing data at end for pair %s, data ends at %s',
                            pair,
                            arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
-        return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing)
+        return parse_ticker_dataframe(pairdata, ticker_interval,
+                                      fill_missing=fill_up_missing,
+                                      drop_incomplete=drop_incomplete)
     else:
         logger.warning(
             f'No history data for pair: "{pair}", interval: {ticker_interval}. '

freqtrade/exchange/exchange.py

@@ -2,23 +2,24 @@
 """
 Cryptocurrency Exchanges support
 """
-import logging
+import asyncio
 import inspect
-from random import randint
-from typing import List, Dict, Tuple, Any, Optional
+import logging
+from copy import deepcopy
 from datetime import datetime
-from math import floor, ceil
+from math import ceil, floor
+from random import randint
+from typing import Any, Dict, List, Optional, Tuple
 
 import arrow
-import asyncio
 import ccxt
 import ccxt.async_support as ccxt_async
 from pandas import DataFrame
 
-from freqtrade import (constants, DependencyException, OperationalException,
-                       TemporaryError, InvalidOrderException)
+from freqtrade import (DependencyException, InvalidOrderException,
+                       OperationalException, TemporaryError, constants)
 from freqtrade.data.converter import parse_ticker_dataframe
-
+from freqtrade.misc import deep_merge_dicts
 
 logger = logging.getLogger(__name__)
 
@@ -68,12 +69,15 @@ class Exchange(object):
     _params: Dict = {}
 
     # Dict to specify which options each exchange implements
-    # TODO: this should be merged with attributes from subclasses
-    # To avoid having to copy/paste this to all subclasses.
-    _ft_has: Dict = {
+    # This defines defaults, which can be selectively overridden by subclasses using _ft_has
+    # or by specifying them in the configuration.
+    _ft_has_default: Dict = {
         "stoploss_on_exchange": False,
         "order_time_in_force": ["gtc"],
+        "ohlcv_candle_limit": 500,
+        "ohlcv_partial_candle": True,
     }
+    _ft_has: Dict = {}
 
     def __init__(self, config: dict) -> None:
         """
@@ -100,6 +104,19 @@ class Exchange(object):
             logger.info('Instance is running with dry_run enabled')
 
         exchange_config = config['exchange']
+
+        # Deep merge ft_has with default ft_has options
+        self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default))
+        if exchange_config.get("_ft_has_params"):
+            self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"),
+                                            self._ft_has)
+            logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has)
+
+        # Assign this directly for easy access
+        self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit']
+        self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle']
+
+        # Initialize ccxt objects
         self._api: ccxt.Exchange = self._init_ccxt(
             exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config'))
         self._api_async: ccxt_async.Exchange = self._init_ccxt(
@@ -506,10 +523,8 @@ class Exchange(object):
     async def _async_get_history(self, pair: str,
                                  ticker_interval: str,
                                  since_ms: int) -> List:
-        # Assume exchange returns 500 candles
-        _LIMIT = 500
 
-        one_call = timeframe_to_msecs(ticker_interval) * _LIMIT
+        one_call = timeframe_to_msecs(ticker_interval) * self._ohlcv_candle_limit
         logger.debug(
             "one_call: %s msecs (%s)",
             one_call,
@@ -566,7 +581,8 @@ class Exchange(object):
                 self._pairs_last_refresh_time[(pair, ticker_interval)] = ticks[-1][0] // 1000
             # keeping parsed dataframe in cache
             self._klines[(pair, ticker_interval)] = parse_ticker_dataframe(
-                ticks, ticker_interval, fill_missing=True)
+                ticks, ticker_interval, fill_missing=True,
+                drop_incomplete=self._ohlcv_partial_candle)
         return tickers
 
     def _now_is_time_to_refresh(self, pair: str, ticker_interval: str) -> bool:

freqtrade/misc.py

@@ -117,6 +117,8 @@ def format_ms_time(date: int) -> str:
 
 def deep_merge_dicts(source, destination):
     """
+    Values from Source override destination, destination is returned (and modified!!)
+    Sample:
     >>> a = { 'first' : { 'rows' : { 'pass' : 'dog', 'number' : '1' } } }
     >>> b = { 'first' : { 'rows' : { 'fail' : 'cat', 'number' : '5' } } }
     >>> merge(b, a) == { 'first' : { 'rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } }

freqtrade/tests/conftest.py

@@ -649,7 +649,7 @@ def ticker_history_list():
 
 @pytest.fixture
 def ticker_history(ticker_history_list):
-    return parse_ticker_dataframe(ticker_history_list, "5m", True)
+    return parse_ticker_dataframe(ticker_history_list, "5m", fill_missing=True)
 
 
 @pytest.fixture
@@ -854,7 +854,7 @@ def tickers():
 @pytest.fixture
 def result():
     with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file:
-        return parse_ticker_dataframe(json.load(data_file), '1m', True)
+        return parse_ticker_dataframe(json.load(data_file), '1m', fill_missing=True)
 
 # FIX:
 # Create an fixture/function

freqtrade/tests/data/test_converter.py

@@ -96,3 +96,50 @@ def test_ohlcv_fill_up_missing_data2(caplog):
 
     assert log_has(f"Missing data fillup: before: {len(data)} - after: {len(data2)}",
                    caplog.record_tuples)
+
+
+def test_ohlcv_drop_incomplete(caplog):
+    ticker_interval = '1d'
+    ticks = [[
+            1559750400000,  # 2019-06-04
+            8.794e-05,  # open
+            8.948e-05,  # high
+            8.794e-05,  # low
+            8.88e-05,  # close
+            2255,  # volume (in quote currency)
+        ],
+        [
+            1559836800000,  # 2019-06-05
+            8.88e-05,
+            8.942e-05,
+            8.88e-05,
+            8.893e-05,
+            9911,
+        ],
+        [
+            1559923200000,  # 2019-06-06
+            8.891e-05,
+            8.893e-05,
+            8.875e-05,
+            8.877e-05,
+            2251
+        ],
+        [
+            1560009600000,  # 2019-06-07
+            8.877e-05,
+            8.883e-05,
+            8.895e-05,
+            8.817e-05,
+            123551
+     ]
+    ]
+    caplog.set_level(logging.DEBUG)
+    data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=False)
+    assert len(data) == 4
+    assert not log_has("Dropping last candle", caplog.record_tuples)
+
+    # Drop last candle
+    data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=True)
+    assert len(data) == 3
+
+    assert log_has("Dropping last candle", caplog.record_tuples)

freqtrade/tests/exchange/test_exchange.py

@@ -1435,3 +1435,30 @@ def test_stoploss_limit_order_dry_run(default_conf, mocker):
     assert order['type'] == order_type
     assert order['price'] == 220
     assert order['amount'] == 1
+
+
+def test_merge_ft_has_dict(default_conf, mocker):
+    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock()))
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    ex = Exchange(default_conf)
+    assert ex._ft_has == Exchange._ft_has_default
+
+    ex = Kraken(default_conf)
+    assert ex._ft_has == Exchange._ft_has_default
+
+    # Binance defines different values
+    ex = Binance(default_conf)
+    assert ex._ft_has != Exchange._ft_has_default
+    assert ex._ft_has['stoploss_on_exchange']
+    assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc']
+
+    conf = copy.deepcopy(default_conf)
+    conf['exchange']['_ft_has_params'] = {"DeadBeef": 20,
+                                          "stoploss_on_exchange": False}
+    # Use settings from configuration (overriding stoploss_on_exchange)
+    ex = Binance(conf)
+    assert ex._ft_has != Exchange._ft_has_default
+    assert not ex._ft_has['stoploss_on_exchange']
+    assert ex._ft_has['DeadBeef'] == 20

freqtrade/tests/strategy/test_interface.py

@@ -111,7 +111,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None:
 
     timerange = TimeRange(None, 'line', 0, -100)
     tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange)
-    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', True)}
+    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)}
     data = strategy.tickerdata_to_dataframe(tickerlist)
     assert len(data['UNITTEST/BTC']) == 102       # partial candle was removed