mirror of
https://github.com/freqtrade/freqtrade.git
synced 2024-09-20 17:41:12 +00:00
Merge pull request #1915 from freqtrade/feat/drop_incomplete_optional
Make dropping the last candle optional (configured per exchange)
This commit is contained in:
Matthias
GitHub
commit
99cceeea70
|
|
@ -301,8 +301,27 @@ This configuration enables binance, as well as rate limiting to avoid bans from
|
|||
|
||||
!!! Note
|
||||
Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings.
|
||||
We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step.
|
||||
We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step.
|
||||
|
||||
#### Advanced FreqTrade Exchange configuration
|
||||
|
||||
Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behaviours.
|
||||
|
||||
Available options are listed in the exchange-class as `_ft_has_default`.
|
||||
|
||||
For example, to test the order type `FOK` with Kraken, and modify candle_limit to 200 (so you only get 200 candles per call):
|
||||
|
||||
```json
|
||||
"exchange": {
|
||||
"name": "kraken",
|
||||
"_ft_has_params": {
|
||||
"order_time_in_force": ["gtc", "fok"],
|
||||
"ohlcv_candle_limit": 200
|
||||
}
|
||||
```
|
||||
|
||||
!!! Warning
|
||||
Please make sure to fully understand the impacts of these settings before modifying them.
|
||||
|
||||
### What values can be used for fiat_display_currency?
|
||||
|
||||
|
|
|
|||
|
|
@ -81,6 +81,51 @@ Please also run `self._validate_whitelist(pairs)` and to check and remove pairs
|
|||
This is a simple method used by `VolumePairList` - however serves as a good example.
|
||||
It implements caching (`@cached(TTLCache(maxsize=1, ttl=1800))`) as well as a configuration option to allow different (but similar) strategies to work with the same PairListProvider.
|
||||
|
||||
## Implement a new Exchange (WIP)
|
||||
|
||||
!!! Note
|
||||
This section is a Work in Progress and is not a complete guide on how to test a new exchange with FreqTrade.
|
||||
|
||||
Most exchanges supported by CCXT should work out of the box.
|
||||
|
||||
### Stoploss On Exchange
|
||||
|
||||
Check if the new exchange supports Stoploss on Exchange orders through their API.
|
||||
|
||||
Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselfs. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects.
|
||||
|
||||
### Incomplete candles
|
||||
|
||||
While fetching OHLCV data, we're may end up getting incomplete candles (Depending on the exchange).
|
||||
To demonstrate this, we'll use daily candles (`"1d"`) to keep things simple.
|
||||
We query the api (`ct.fetch_ohlcv()`) for the timeframe and look at the date of the last entry. If this entry changes or shows the date of a "incomplete" candle, then we should drop this since having incomplete candles is problematic because indicators assume that only complete candles are passed to them, and will generate a lot of false buy signals. By default, we're therefore removing the last candle assuming it's incomplete.
|
||||
|
||||
To check how the new exchange behaves, you can use the following snippet:
|
||||
|
||||
``` python
|
||||
import ccxt
|
||||
from datetime import datetime
|
||||
from freqtrade.data.converter import parse_ticker_dataframe
|
||||
ct = ccxt.binance()
|
||||
timeframe = "1d"
|
||||
pair = "XLM/BTC" # Make sure to use a pair that exists on that exchange!
|
||||
raw = ct.fetch_ohlcv(pair, timeframe=timeframe)
|
||||
|
||||
# convert to dataframe
|
||||
df1 = parse_ticker_dataframe(raw, timeframe, drop_incomplete=False)
|
||||
|
||||
print(df1["date"].tail(1))
|
||||
print(datetime.utcnow())
|
||||
```
|
||||
|
||||
``` output
|
||||
19 2019-06-08 00:00:00+00:00
|
||||
2019-06-09 12:30:27.873327
|
||||
```
|
||||
|
||||
The output will show the last entry from the Exchange as well as the current UTC date.
|
||||
If the day shows the same day, then the last candle can be assumed as incomplete and should be dropped (leave the setting `"ohlcv_partial_candle"` from the exchange-class untouched / True). Otherwise, set `"ohlcv_partial_candle"` to `False` to not drop Candles (shown in the example above).
|
||||
|
||||
## Creating a release
|
||||
|
||||
This part of the documentation is aimed at maintainers, and shows how to create a release.
|
||||
|
|
|
|||
|
|
@ -10,14 +10,16 @@ from pandas import DataFrame, to_datetime
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_ticker_dataframe(ticker: list, ticker_interval: str,
|
||||
fill_missing: bool = True) -> DataFrame:
|
||||
def parse_ticker_dataframe(ticker: list, ticker_interval: str, *,
|
||||
fill_missing: bool = True,
|
||||
drop_incomplete: bool = True) -> DataFrame:
|
||||
"""
|
||||
Converts a ticker-list (format ccxt.fetch_ohlcv) to a Dataframe
|
||||
:param ticker: ticker list, as returned by exchange.async_get_candle_history
|
||||
:param ticker_interval: ticker_interval (e.g. 5m). Used to fill up eventual missing data
|
||||
:param fill_missing: fill up missing candles with 0 candles
|
||||
(see ohlcv_fill_up_missing_data for details)
|
||||
:param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete
|
||||
:return: DataFrame
|
||||
"""
|
||||
logger.debug("Parsing tickerlist to dataframe")
|
||||
|
|
@ -43,8 +45,10 @@ def parse_ticker_dataframe(ticker: list, ticker_interval: str,
|
|||
'close': 'last',
|
||||
'volume': 'max',
|
||||
})
|
||||
frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle
|
||||
logger.debug('Dropping last candle')
|
||||
# eliminate partial candle
|
||||
if drop_incomplete:
|
||||
frame.drop(frame.tail(1).index, inplace=True)
|
||||
logger.debug('Dropping last candle')
|
||||
|
||||
if fill_missing:
|
||||
return ohlcv_fill_up_missing_data(frame, ticker_interval)
|
||||
|
|
|
|||
|
|
@ -81,10 +81,20 @@ def load_pair_history(pair: str,
|
|||
timerange: TimeRange = TimeRange(None, None, 0, 0),
|
||||
refresh_pairs: bool = False,
|
||||
exchange: Optional[Exchange] = None,
|
||||
fill_up_missing: bool = True
|
||||
fill_up_missing: bool = True,
|
||||
drop_incomplete: bool = True
|
||||
) -> DataFrame:
|
||||
"""
|
||||
Loads cached ticker history for the given pair.
|
||||
:param pair: Pair to load data for
|
||||
:param ticker_interval: Ticker-interval (e.g. "5m")
|
||||
:param datadir: Path to the data storage location.
|
||||
:param timerange: Limit data to be loaded to this timerange
|
||||
:param refresh_pairs: Refresh pairs from exchange.
|
||||
(Note: Requires exchange to be passed as well.)
|
||||
:param exchange: Exchange object (needed when using "refresh_pairs")
|
||||
:param fill_up_missing: Fill missing values with "No action"-candles
|
||||
:param drop_incomplete: Drop last candle assuming it may be incomplete.
|
||||
:return: DataFrame with ohlcv data
|
||||
"""
|
||||
|
||||
|
|
@ -106,7 +116,9 @@ def load_pair_history(pair: str,
|
|||
logger.warning('Missing data at end for pair %s, data ends at %s',
|
||||
pair,
|
||||
arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
|
||||
return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing)
|
||||
return parse_ticker_dataframe(pairdata, ticker_interval,
|
||||
fill_missing=fill_up_missing,
|
||||
drop_incomplete=drop_incomplete)
|
||||
else:
|
||||
logger.warning(
|
||||
f'No history data for pair: "{pair}", interval: {ticker_interval}. '
|
||||
|
|
|
|||
|
|
@ -2,23 +2,24 @@
|
|||
"""
|
||||
Cryptocurrency Exchanges support
|
||||
"""
|
||||
import logging
|
||||
import asyncio
|
||||
import inspect
|
||||
from random import randint
|
||||
from typing import List, Dict, Tuple, Any, Optional
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
from datetime import datetime
|
||||
from math import floor, ceil
|
||||
from math import ceil, floor
|
||||
from random import randint
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import arrow
|
||||
import asyncio
|
||||
import ccxt
|
||||
import ccxt.async_support as ccxt_async
|
||||
from pandas import DataFrame
|
||||
|
||||
from freqtrade import (constants, DependencyException, OperationalException,
|
||||
TemporaryError, InvalidOrderException)
|
||||
from freqtrade import (DependencyException, InvalidOrderException,
|
||||
OperationalException, TemporaryError, constants)
|
||||
from freqtrade.data.converter import parse_ticker_dataframe
|
||||
|
||||
from freqtrade.misc import deep_merge_dicts
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -68,12 +69,15 @@ class Exchange(object):
|
|||
_params: Dict = {}
|
||||
|
||||
# Dict to specify which options each exchange implements
|
||||
# TODO: this should be merged with attributes from subclasses
|
||||
# To avoid having to copy/paste this to all subclasses.
|
||||
_ft_has: Dict = {
|
||||
# This defines defaults, which can be selectively overridden by subclasses using _ft_has
|
||||
# or by specifying them in the configuration.
|
||||
_ft_has_default: Dict = {
|
||||
"stoploss_on_exchange": False,
|
||||
"order_time_in_force": ["gtc"],
|
||||
"ohlcv_candle_limit": 500,
|
||||
"ohlcv_partial_candle": True,
|
||||
}
|
||||
_ft_has: Dict = {}
|
||||
|
||||
def __init__(self, config: dict) -> None:
|
||||
"""
|
||||
|
|
@ -100,6 +104,19 @@ class Exchange(object):
|
|||
logger.info('Instance is running with dry_run enabled')
|
||||
|
||||
exchange_config = config['exchange']
|
||||
|
||||
# Deep merge ft_has with default ft_has options
|
||||
self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default))
|
||||
if exchange_config.get("_ft_has_params"):
|
||||
self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"),
|
||||
self._ft_has)
|
||||
logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has)
|
||||
|
||||
# Assign this directly for easy access
|
||||
self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit']
|
||||
self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle']
|
||||
|
||||
# Initialize ccxt objects
|
||||
self._api: ccxt.Exchange = self._init_ccxt(
|
||||
exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config'))
|
||||
self._api_async: ccxt_async.Exchange = self._init_ccxt(
|
||||
|
|
@ -506,10 +523,8 @@ class Exchange(object):
|
|||
async def _async_get_history(self, pair: str,
|
||||
ticker_interval: str,
|
||||
since_ms: int) -> List:
|
||||
# Assume exchange returns 500 candles
|
||||
_LIMIT = 500
|
||||
|
||||
one_call = timeframe_to_msecs(ticker_interval) * _LIMIT
|
||||
one_call = timeframe_to_msecs(ticker_interval) * self._ohlcv_candle_limit
|
||||
logger.debug(
|
||||
"one_call: %s msecs (%s)",
|
||||
one_call,
|
||||
|
|
@ -566,7 +581,8 @@ class Exchange(object):
|
|||
self._pairs_last_refresh_time[(pair, ticker_interval)] = ticks[-1][0] // 1000
|
||||
# keeping parsed dataframe in cache
|
||||
self._klines[(pair, ticker_interval)] = parse_ticker_dataframe(
|
||||
ticks, ticker_interval, fill_missing=True)
|
||||
ticks, ticker_interval, fill_missing=True,
|
||||
drop_incomplete=self._ohlcv_partial_candle)
|
||||
return tickers
|
||||
|
||||
def _now_is_time_to_refresh(self, pair: str, ticker_interval: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -117,6 +117,8 @@ def format_ms_time(date: int) -> str:
|
|||
|
||||
def deep_merge_dicts(source, destination):
|
||||
"""
|
||||
Values from Source override destination, destination is returned (and modified!!)
|
||||
Sample:
|
||||
>>> a = { 'first' : { 'rows' : { 'pass' : 'dog', 'number' : '1' } } }
|
||||
>>> b = { 'first' : { 'rows' : { 'fail' : 'cat', 'number' : '5' } } }
|
||||
>>> merge(b, a) == { 'first' : { 'rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } }
|
||||
|
|
|
|||
|
|
@ -649,7 +649,7 @@ def ticker_history_list():
|
|||
|
||||
@pytest.fixture
|
||||
def ticker_history(ticker_history_list):
|
||||
return parse_ticker_dataframe(ticker_history_list, "5m", True)
|
||||
return parse_ticker_dataframe(ticker_history_list, "5m", fill_missing=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -854,7 +854,7 @@ def tickers():
|
|||
@pytest.fixture
|
||||
def result():
|
||||
with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file:
|
||||
return parse_ticker_dataframe(json.load(data_file), '1m', True)
|
||||
return parse_ticker_dataframe(json.load(data_file), '1m', fill_missing=True)
|
||||
|
||||
# FIX:
|
||||
# Create an fixture/function
|
||||
|
|
|
|||
|
|
@ -96,3 +96,50 @@ def test_ohlcv_fill_up_missing_data2(caplog):
|
|||
|
||||
assert log_has(f"Missing data fillup: before: {len(data)} - after: {len(data2)}",
|
||||
caplog.record_tuples)
|
||||
|
||||
|
||||
def test_ohlcv_drop_incomplete(caplog):
|
||||
ticker_interval = '1d'
|
||||
ticks = [[
|
||||
1559750400000, # 2019-06-04
|
||||
8.794e-05, # open
|
||||
8.948e-05, # high
|
||||
8.794e-05, # low
|
||||
8.88e-05, # close
|
||||
2255, # volume (in quote currency)
|
||||
],
|
||||
[
|
||||
1559836800000, # 2019-06-05
|
||||
8.88e-05,
|
||||
8.942e-05,
|
||||
8.88e-05,
|
||||
8.893e-05,
|
||||
9911,
|
||||
],
|
||||
[
|
||||
1559923200000, # 2019-06-06
|
||||
8.891e-05,
|
||||
8.893e-05,
|
||||
8.875e-05,
|
||||
8.877e-05,
|
||||
2251
|
||||
],
|
||||
[
|
||||
1560009600000, # 2019-06-07
|
||||
8.877e-05,
|
||||
8.883e-05,
|
||||
8.895e-05,
|
||||
8.817e-05,
|
||||
123551
|
||||
]
|
||||
]
|
||||
caplog.set_level(logging.DEBUG)
|
||||
data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=False)
|
||||
assert len(data) == 4
|
||||
assert not log_has("Dropping last candle", caplog.record_tuples)
|
||||
|
||||
# Drop last candle
|
||||
data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=True)
|
||||
assert len(data) == 3
|
||||
|
||||
assert log_has("Dropping last candle", caplog.record_tuples)
|
||||
|
|
|
|||
|
|
@ -1435,3 +1435,30 @@ def test_stoploss_limit_order_dry_run(default_conf, mocker):
|
|||
assert order['type'] == order_type
|
||||
assert order['price'] == 220
|
||||
assert order['amount'] == 1
|
||||
|
||||
|
||||
def test_merge_ft_has_dict(default_conf, mocker):
|
||||
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock()))
|
||||
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
|
||||
ex = Exchange(default_conf)
|
||||
assert ex._ft_has == Exchange._ft_has_default
|
||||
|
||||
ex = Kraken(default_conf)
|
||||
assert ex._ft_has == Exchange._ft_has_default
|
||||
|
||||
# Binance defines different values
|
||||
ex = Binance(default_conf)
|
||||
assert ex._ft_has != Exchange._ft_has_default
|
||||
assert ex._ft_has['stoploss_on_exchange']
|
||||
assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc']
|
||||
|
||||
conf = copy.deepcopy(default_conf)
|
||||
conf['exchange']['_ft_has_params'] = {"DeadBeef": 20,
|
||||
"stoploss_on_exchange": False}
|
||||
# Use settings from configuration (overriding stoploss_on_exchange)
|
||||
ex = Binance(conf)
|
||||
assert ex._ft_has != Exchange._ft_has_default
|
||||
assert not ex._ft_has['stoploss_on_exchange']
|
||||
assert ex._ft_has['DeadBeef'] == 20
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None:
|
|||
|
||||
timerange = TimeRange(None, 'line', 0, -100)
|
||||
tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange)
|
||||
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', True)}
|
||||
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)}
|
||||
data = strategy.tickerdata_to_dataframe(tickerlist)
|
||||
assert len(data['UNITTEST/BTC']) == 102 # partial candle was removed
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user