diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index fc5f11d50..88f183a4a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -7,11 +7,23 @@ import argparse import logging import re import arrow -from typing import List, Tuple, Optional +from typing import List, Optional, NamedTuple from freqtrade import __version__, constants +class TimeRange(NamedTuple): + """ + NamedTuple Defining timerange inputs. + [start/stop]type defines if [start/stop]ts shall be used. + if *type is none, don't use corresponding startvalue. + """ + starttype: Optional[str] = None + stoptype: Optional[str] = None + startts: int = 0 + stopts: int = 0 + + class Arguments(object): """ Arguments Class. Manage the arguments received by the cli @@ -224,15 +236,14 @@ class Arguments(object): self.hyperopt_options(hyperopt_cmd) @staticmethod - def parse_timerange(text: Optional[str]) -> \ - Optional[Tuple[Tuple, Optional[int], Optional[int]]]: + def parse_timerange(text: Optional[str]) -> TimeRange: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange :return: Start and End range period """ if text is None: - return None + return TimeRange() syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), @@ -248,8 +259,8 @@ class Arguments(object): if match: # Regex has matched rvals = match.groups() index = 0 - start: Optional[int] = None - stop: Optional[int] = None + start: int = 0 + stop: int = 0 if stype[0]: starts = rvals[index] if stype[0] == 'date': @@ -265,7 +276,7 @@ class Arguments(object): else arrow.get(stops, 'YYYYMMDD').timestamp else: stop = int(stops) - return stype, start, stop + return TimeRange(stype[0], stype[1], start, stop) raise Exception('Incorrect syntax for timerange "%s"' % text) def scripts_options(self) -> None: diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 3c67db71f..5e768518b 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -18,6 +18,8 @@ _API: ccxt.Exchange = None _CONF: Dict = {} API_RETRY_COUNT = 4 +_CACHED_TICKER: Dict[str, Any] = {} + # Holds all open sell orders for dry_run _DRY_RUN_OPEN_ORDERS: Dict[str, Any] = {} @@ -264,17 +266,29 @@ def get_tickers() -> Dict: raise OperationalException(e) -# TODO: remove refresh argument, keeping it to keep track of where it was intended to be used @retrier def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict: - try: - return _API.fetch_ticker(pair) - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - 'Could not load ticker history due to {}. Message: {}'.format( - e.__class__.__name__, e)) - except ccxt.BaseError as e: - raise OperationalException(e) + global _CACHED_TICKER + if refresh or pair not in _CACHED_TICKER.keys(): + try: + data = _API.fetch_ticker(pair) + try: + _CACHED_TICKER[pair] = { + 'bid': float(data['bid']), + 'ask': float(data['ask']), + } + except KeyError as e: + logger.debug("Could not cache ticker data for %s", pair) + return data + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not load ticker history due to {}. Message: {}'.format( + e.__class__.__name__, e)) + except ccxt.BaseError as e: + raise OperationalException(e) + else: + logger.info("returning cached ticker-data for %s", pair) + return _CACHED_TICKER[pair] @retrier diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 711adfd28..00f05cc46 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -9,39 +9,40 @@ import arrow from freqtrade import misc, constants from freqtrade.exchange import get_ticker_history +from freqtrade.arguments import TimeRange from user_data.hyperopt_conf import hyperopt_optimize_conf logger = logging.getLogger(__name__) -def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) -> List[Dict]: +def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: if not tickerlist: return tickerlist - stype, start, stop = timerange - start_index = 0 stop_index = len(tickerlist) - if stype[0] == 'line': - stop_index = start - if stype[0] == 'index': - start_index = start - elif stype[0] == 'date': - while start_index < len(tickerlist) and tickerlist[start_index][0] < start * 1000: + if timerange.starttype == 'line': + stop_index = timerange.startts + if timerange.starttype == 'index': + start_index = timerange.startts + elif timerange.starttype == 'date': + while (start_index < len(tickerlist) and + tickerlist[start_index][0] < timerange.startts * 1000): start_index += 1 - if stype[1] == 'line': - start_index = len(tickerlist) + stop - if stype[1] == 'index': - stop_index = stop - elif stype[1] == 'date': - while stop_index > 0 and tickerlist[stop_index-1][0] > stop * 1000: + if timerange.stoptype == 'line': + start_index = len(tickerlist) + timerange.stopts + if timerange.stoptype == 'index': + stop_index = timerange.stopts + elif timerange.stoptype == 'date': + while (stop_index > 0 and + tickerlist[stop_index-1][0] > timerange.stopts * 1000): stop_index -= 1 if start_index > stop_index: - raise ValueError(f'The timerange [{start},{stop}] is incorrect') + raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') return tickerlist[start_index:stop_index] @@ -49,7 +50,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) - def load_tickerdata_file( datadir: str, pair: str, ticker_interval: str, - timerange: Optional[Tuple[Tuple, int, int]] = None) -> Optional[List[Dict]]: + timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]: """ Load a pair from file, :return dict OR empty if unsuccesful @@ -84,7 +85,7 @@ def load_data(datadir: str, ticker_interval: str, pairs: Optional[List[str]] = None, refresh_pairs: Optional[bool] = False, - timerange: Optional[Tuple[Tuple, int, int]] = None) -> Dict[str, List]: + timerange: TimeRange = TimeRange()) -> Dict[str, List]: """ Loads ticker history data for the given parameters :return: dict @@ -124,7 +125,7 @@ def make_testdata_path(datadir: str) -> str: def download_pairs(datadir, pairs: List[str], ticker_interval: str, - timerange: Optional[Tuple[Tuple, int, int]] = None) -> bool: + timerange: TimeRange = TimeRange()) -> bool: """For each pairs passed in parameters, download the ticker intervals""" for pair in pairs: try: @@ -144,7 +145,7 @@ def download_pairs(datadir, pairs: List[str], def load_cached_data_for_updating(filename: str, tick_interval: str, - timerange: Optional[Tuple[Tuple, int, int]]) -> Tuple[ + timerange: Optional[TimeRange]) -> Tuple[ List[Any], Optional[int]]: """ @@ -155,10 +156,10 @@ def load_cached_data_for_updating(filename: str, # user sets timerange, so find the start time if timerange: - if timerange[0][0] == 'date': - since_ms = timerange[1] * 1000 - elif timerange[0][1] == 'line': - num_minutes = timerange[2] * constants.TICKER_INTERVAL_MINUTES[tick_interval] + if timerange.starttype == 'date': + since_ms = timerange.startts * 1000 + elif timerange.stoptype == 'line': + num_minutes = timerange.stopts * constants.TICKER_INTERVAL_MINUTES[tick_interval] since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file @@ -188,7 +189,7 @@ def load_cached_data_for_updating(filename: str, def download_backtesting_testdata(datadir: str, pair: str, tick_interval: str = '5m', - timerange: Optional[Tuple[Tuple, int, int]] = None) -> None: + timerange: Optional[TimeRange] = None) -> None: """ Download the latest ticker intervals from the exchange for the pairs passed in parameters diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d7ed45955..3dd643561 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -221,7 +221,7 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = optimize.load_data( # type: ignore # timerange will be refactored + data = optimize.load_data( self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 74b39b445..878acc2dc 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -497,7 +497,7 @@ class Hyperopt(Backtesting): def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = load_data( # type: ignore # timerange will be refactored + data = load_data( datadir=str(self.config.get('datadir')), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 56812c75e..97a723929 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -310,9 +310,19 @@ def test_get_ticker(default_conf, mocker): # if not fetching a new result we should get the cached ticker ticker = get_ticker(pair='ETH/BTC') + assert api_mock.fetch_ticker.call_count == 1 assert ticker['bid'] == 0.5 assert ticker['ask'] == 1 + assert 'ETH/BTC' in exchange._CACHED_TICKER + assert exchange._CACHED_TICKER['ETH/BTC']['bid'] == 0.5 + assert exchange._CACHED_TICKER['ETH/BTC']['ask'] == 1 + + # Test caching + api_mock.fetch_ticker = MagicMock() + get_ticker(pair='ETH/BTC', refresh=False) + assert api_mock.fetch_ticker.call_count == 0 + with pytest.raises(TemporaryError): # test retrier api_mock.fetch_ticker = MagicMock(side_effect=ccxt.NetworkError) mocker.patch('freqtrade.exchange._API', api_mock) @@ -323,6 +333,10 @@ def test_get_ticker(default_conf, mocker): mocker.patch('freqtrade.exchange._API', api_mock) get_ticker(pair='ETH/BTC', refresh=True) + api_mock.fetch_ticker = MagicMock(return_value={}) + mocker.patch('freqtrade.exchange._API', api_mock) + get_ticker(pair='ETH/BTC', refresh=True) + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 65820ac09..efcee3839 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -13,7 +13,7 @@ from arrow import Arrow from freqtrade import optimize from freqtrade.analyze import Analyze -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration from freqtrade.tests.conftest import log_has @@ -30,7 +30,7 @@ def trim_dictlist(dict_list, num): def load_data_test(what): - timerange = ((None, 'line'), None, -100) + timerange = TimeRange(None, 'line', 0, -100) data = optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC'], timerange=timerange) pair = data['UNITTEST/BTC'] @@ -311,7 +311,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: Test Backtesting.tickerdata_to_dataframe() method """ mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - timerange = ((None, 'line'), None, -100) + timerange = TimeRange(None, 'line', 0, -100) tick = optimize.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': tick} diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 349fa3be3..3f358cfb8 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -11,6 +11,7 @@ from freqtrade.misc import file_dump_json from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \ download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, \ load_cached_data_for_updating +from freqtrade.arguments import TimeRange from freqtrade.tests.conftest import log_has # Change this if modifying UNITTEST/BTC testdatafile @@ -176,7 +177,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts earlier than the cached data # should fully update data - timerange = (('date', None), test_data[0][0] / 1000 - 1, None) + timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -187,13 +188,13 @@ def test_load_cached_data_for_updating(mocker) -> None: num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 data, start_ts = load_cached_data_for_updating(test_filename, '1m', - ((None, 'line'), None, -num_lines)) + TimeRange(None, 'line', 0, -num_lines)) assert data == [] assert start_ts < test_data[0][0] - 1 # timeframe starts in the center of the cached data # should return the chached data w/o the last item - timerange = (('date', None), test_data[0][0] / 1000 + 1, None) + timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -202,7 +203,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -211,7 +212,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts after the chached data # should return the chached data w/o the last item - timerange = (('date', None), test_data[-1][0] / 1000 + 1, None) + timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -220,7 +221,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -230,7 +231,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # no timeframe is set # should return the chached data w/o the last item num_lines = 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -239,7 +240,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # no datafile exist # should return timestamp start time - timerange = (('date', None), now_ts - 10000, None) + timerange = TimeRange('date', None, now_ts - 10000, 0) data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', '1m', timerange) @@ -248,7 +249,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', '1m', timerange) @@ -343,7 +344,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(-\d+)$ # This pattern uses the latest N elements - timerange = ((None, 'line'), None, -5) + timerange = TimeRange(None, 'line', 0, -5) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -353,7 +354,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d+)-$ # This pattern keep X element from the end - timerange = (('line', None), 5, None) + timerange = TimeRange('line', None, 5, 0) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -363,7 +364,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d+)-(\d+)$ # This pattern extract a window - timerange = (('index', 'index'), 5, 10) + timerange = TimeRange('index', 'index', 5, 10) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -374,7 +375,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d{8})-(\d{8})$ # This pattern extract a window between the dates - timerange = (('date', 'date'), ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) + timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -385,7 +386,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^-(\d{8})$ # This pattern extracts elements from the start to the date - timerange = ((None, 'date'), None, ticker_list[10][0] / 1000 - 1) + timerange = TimeRange(None, 'date', 0, ticker_list[10][0] / 1000 - 1) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -395,7 +396,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d{8})-$ # This pattern extracts elements from the date to now - timerange = (('date', None), ticker_list[10][0] / 1000 - 1, None) + timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, None) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -405,7 +406,7 @@ def test_trim_tickerlist() -> None: # Test a wrong pattern # This pattern must return the list unchanged - timerange = ((None, None), None, 5) + timerange = TimeRange(None, None, None, 5) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 01033ce7d..418f31851 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -13,6 +13,7 @@ from pandas import DataFrame from freqtrade.analyze import Analyze, SignalType from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.arguments import TimeRange from freqtrade.tests.conftest import log_has # Avoid to reinit the same object again and again @@ -183,7 +184,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None: """ analyze = Analyze(default_conf) - timerange = ((None, 'line'), None, -100) + timerange = TimeRange(None, 'line', 0, -100) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': tick} data = analyze.tickerdata_to_dataframe(tickerlist) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 474aa2507..6c3ecb913 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -9,7 +9,7 @@ import logging import pytest -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange def test_arguments_object() -> None: @@ -107,20 +107,24 @@ def test_parse_args_dynamic_whitelist_invalid_values() -> None: def test_parse_timerange_incorrect() -> None: - assert ((None, 'line'), None, -200) == Arguments.parse_timerange('-200') - assert (('line', None), 200, None) == Arguments.parse_timerange('200-') - assert (('index', 'index'), 200, 500) == Arguments.parse_timerange('200-500') + assert TimeRange(None, 'line', 0, -200) == Arguments.parse_timerange('-200') + assert TimeRange('line', None, 200, 0) == Arguments.parse_timerange('200-') + assert TimeRange('index', 'index', 200, 500) == Arguments.parse_timerange('200-500') - assert (('date', None), 1274486400, None) == Arguments.parse_timerange('20100522-') - assert ((None, 'date'), None, 1274486400) == Arguments.parse_timerange('-20100522') + assert TimeRange('date', None, 1274486400, 0) == Arguments.parse_timerange('20100522-') + assert TimeRange(None, 'date', 0, 1274486400) == Arguments.parse_timerange('-20100522') timerange = Arguments.parse_timerange('20100522-20150730') - assert timerange == (('date', 'date'), 1274486400, 1438214400) + assert timerange == TimeRange('date', 'date', 1274486400, 1438214400) # Added test for unix timestamp - BTC genesis date - assert (('date', None), 1231006505, None) == Arguments.parse_timerange('1231006505-') - assert ((None, 'date'), None, 1233360000) == Arguments.parse_timerange('-1233360000') + assert TimeRange('date', None, 1231006505, 0) == Arguments.parse_timerange('1231006505-') + assert TimeRange(None, 'date', 0, 1233360000) == Arguments.parse_timerange('-1233360000') timerange = Arguments.parse_timerange('1231006505-1233360000') - assert timerange == (('date', 'date'), 1231006505, 1233360000) + assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange + + # TODO: Find solution for the following case (passing timestamp in ms) + timerange = Arguments.parse_timerange('1231006505000-1233360000000') + assert TimeRange('date', 'date', 1231006505, 1233360000) != timerange with pytest.raises(Exception, match=r'Incorrect syntax.*'): Arguments.parse_timerange('-') diff --git a/requirements.txt b/requirements.txt index 6e7550515..5f5183321 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.121 +ccxt==1.14.155 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 @@ -10,14 +10,14 @@ pandas==0.23.0 scikit-learn==0.19.1 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.14.3 +numpy==1.14.4 TA-Lib==0.4.17 -pytest==3.6.0 +pytest==3.6.1 pytest-mock==1.10.0 pytest-cov==2.5.1 hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 -networkx==1.11 +networkx==1.11 # pyup: ignore tabulate==0.8.2 coinmarketcap==5.0.3