Merge branch 'develop' into timeframe_use_ccxt

docs/requirements-docs.txt

@@ -1 +1 @@
-mkdocs-material==4.4.0
+mkdocs-material==4.4.1

freqtrade/__init__.py

@@ -11,7 +11,7 @@ class DependencyException(Exception):
 
 class OperationalException(Exception):
     """
-    Requires manual intervention.
+    Requires manual intervention and will usually stop the bot.
     This happens when an exchange returns an unexpected error during runtime
     or given configuration is invalid.
     """

freqtrade/data/history.py

@@ -280,6 +280,35 @@ def download_pair_history(datadir: Optional[Path],
         return False
 
 
+def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
+                                dl_path: Path, timerange: TimeRange,
+                                erase=False) -> List[str]:
+    """
+    Refresh stored ohlcv data for backtesting and hyperopt operations.
+    Used by freqtrade download-data
+    :return: Pairs not available
+    """
+    pairs_not_available = []
+    for pair in pairs:
+        if pair not in exchange.markets:
+            pairs_not_available.append(pair)
+            logger.info(f"Skipping pair {pair}...")
+            continue
+        for ticker_interval in timeframes:
+
+            dl_file = pair_data_filename(dl_path, pair, ticker_interval)
+            if erase and dl_file.exists():
+                logger.info(
+                    f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
+                dl_file.unlink()
+
+            logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
+            download_pair_history(datadir=dl_path, exchange=exchange,
+                                  pair=pair, ticker_interval=str(ticker_interval),
+                                  timerange=timerange)
+    return pairs_not_available
+
+
 def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
     """
     Get the maximum timeframe for the given backtest data

freqtrade/exchange/binance.py

@@ -2,6 +2,9 @@
 import logging
 from typing import Dict
 
+import ccxt
+
+from freqtrade import DependencyException, OperationalException, TemporaryError
 from freqtrade.exchange import Exchange
 
 logger = logging.getLogger(__name__)
@@ -25,3 +28,53 @@ class Binance(Exchange):
         limit = min(list(filter(lambda x: limit <= x, limit_range)))
 
         return super().get_order_book(pair, limit)
+
+    def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict:
+        """
+        creates a stoploss limit order.
+        this stoploss-limit is binance-specific.
+        It may work with a limited number of other exchanges, but this has not been tested yet.
+
+        """
+        ordertype = "stop_loss_limit"
+
+        stop_price = self.symbol_price_prec(pair, stop_price)
+
+        # Ensure rate is less than stop price
+        if stop_price <= rate:
+            raise OperationalException(
+                'In stoploss limit order, stop price should be more than limit price')
+
+        if self._config['dry_run']:
+            dry_order = self.dry_run_order(
+                pair, ordertype, "sell", amount, stop_price)
+            return dry_order
+
+        try:
+            params = self._params.copy()
+            params.update({'stopPrice': stop_price})
+
+            amount = self.symbol_amount_prec(pair, amount)
+
+            rate = self.symbol_price_prec(pair, rate)
+
+            order = self._api.create_order(pair, ordertype, 'sell',
+                                           amount, rate, params)
+            logger.info('stoploss limit order added for %s. '
+                        'stop price: %s. limit: %s', pair, stop_price, rate)
+            return order
+        except ccxt.InsufficientFunds as e:
+            raise DependencyException(
+                f'Insufficient funds to create {ordertype} sell order on market {pair}.'
+                f'Tried to sell amount {amount} at rate {rate}.'
+                f'Message: {e}') from e
+        except ccxt.InvalidOrder as e:
+            raise DependencyException(
+                f'Could not create {ordertype} sell order on market {pair}. '
+                f'Tried to sell amount {amount} at rate {rate}.'
+                f'Message: {e}') from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e

freqtrade/exchange/exchange.py

@@ -321,7 +321,7 @@ class Exchange(object):
         if (order_types.get("stoploss_on_exchange")
                 and not self._ft_has.get("stoploss_on_exchange", False)):
             raise OperationalException(
-                'On exchange stoploss is not supported for %s.' % self.name
+                f'On exchange stoploss is not supported for {self.name}.'
             )
 
     def validate_order_time_in_force(self, order_time_in_force: Dict) -> None:
@@ -451,30 +451,14 @@ class Exchange(object):
     def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict:
         """
         creates a stoploss limit order.
-        NOTICE: it is not supported by all exchanges. only binance is tested for now.
-        TODO: implementation maybe needs to be moved to the binance subclass
+        Since ccxt does not unify stoploss-limit orders yet, this needs to be implemented in each
+        exchange's subclass.
+        The exception below should never raise, since we disallow
+        starting the bot in validate_ordertypes()
+        Note: Changes to this interface need to be applied to all sub-classes too.
         """
-        ordertype = "stop_loss_limit"
 
-        stop_price = self.symbol_price_prec(pair, stop_price)
-
-        # Ensure rate is less than stop price
-        if stop_price <= rate:
-            raise OperationalException(
-                'In stoploss limit order, stop price should be more than limit price')
-
-        if self._config['dry_run']:
-            dry_order = self.dry_run_order(
-                pair, ordertype, "sell", amount, stop_price)
-            return dry_order
-
-        params = self._params.copy()
-        params.update({'stopPrice': stop_price})
-
-        order = self.create_order(pair, ordertype, 'sell', amount, rate, params)
-        logger.info('stoploss limit order added for %s. '
-                    'stop price: %s. limit: %s', pair, stop_price, rate)
-        return order
+        raise OperationalException(f"stoploss_limit is not implemented for {self.name}.")
 
     @retrier
     def get_balance(self, currency: str) -> float:

freqtrade/freqtradebot.py

@@ -216,7 +216,7 @@ class FreqtradeBot(object):
         if stake_amount == constants.UNLIMITED_STAKE_AMOUNT:
             open_trades = len(Trade.get_open_trades())
             if open_trades >= self.config['max_open_trades']:
-                logger.warning('Can\'t open a new trade: max number of trades is reached')
+                logger.warning("Can't open a new trade: max number of trades is reached")
                 return None
             return available_amount / (self.config['max_open_trades'] - open_trades)
 
@@ -351,8 +351,8 @@ class FreqtradeBot(object):
         min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested)
         if min_stake_amount is not None and min_stake_amount > stake_amount:
             logger.warning(
-                f'Can\'t open a new trade for {pair_s}: stake amount '
-                f'is too small ({stake_amount} < {min_stake_amount})'
+                f"Can't open a new trade for {pair_s}: stake amount "
+                f"is too small ({stake_amount} < {min_stake_amount})"
             )
             return False
 

freqtrade/optimize/hyperopt.py

@@ -124,14 +124,14 @@ class Hyperopt:
         Save hyperopt trials to file
         """
         if self.trials:
-            logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file)
+            logger.info("Saving %d evaluations to '%s'", len(self.trials), self.trials_file)
             dump(self.trials, self.trials_file)
 
     def read_trials(self) -> List:
         """
         Read hyperopt trials file
         """
-        logger.info('Reading Trials from \'%s\'', self.trials_file)
+        logger.info("Reading Trials from '%s'", self.trials_file)
         trials = load(self.trials_file)
         self.trials_file.unlink()
         return trials
@@ -379,7 +379,7 @@ class Hyperopt:
         self.load_previous_results()
 
         cpus = cpu_count()
-        logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!')
+        logger.info(f"Found {cpus} CPU cores. Let's make them scream!")
         config_jobs = self.config.get('hyperopt_jobs', -1)
         logger.info(f'Number of parallel jobs set as: {config_jobs}')
 

freqtrade/persistence.py

@@ -48,8 +48,8 @@ def init(db_url: str, clean_open_orders: bool = False) -> None:
     try:
         engine = create_engine(db_url, **kwargs)
     except NoSuchModuleError:
-        raise OperationalException(f'Given value for db_url: \'{db_url}\' '
-                                   f'is no valid database URL! (See {_SQL_DOCS_URL})')
+        raise OperationalException(f"Given value for db_url: '{db_url}' "
+                                   f"is no valid database URL! (See {_SQL_DOCS_URL})")
 
     session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
     Trade.session = session()

freqtrade/tests/data/test_history.py

@@ -5,7 +5,7 @@ import os
 import uuid
 from pathlib import Path
 from shutil import copyfile
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, PropertyMock
 
 import arrow
 import pytest
@@ -17,6 +17,7 @@ from freqtrade.data import history
 from freqtrade.data.history import (download_pair_history,
                                     load_cached_data_for_updating,
                                     load_tickerdata_file, make_testdata_path,
+                                    refresh_backtest_ohlcv_data,
                                     trim_tickerlist)
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
@@ -558,3 +559,43 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None:
     assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
                                               min_date, max_date, timeframe_to_minutes('5m'))
     assert len(caplog.record_tuples) == 0
+
+
+def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog):
+    dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
+    )
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    mocker.patch.object(Path, "unlink", MagicMock())
+
+    ex = get_patched_exchange(mocker, default_conf)
+    timerange = TimeRange.parse_timerange("20190101-20190102")
+    refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
+                                timeframes=["1m", "5m"], dl_path=make_testdata_path(None),
+                                timerange=timerange, erase=True
+                                )
+
+    assert dl_mock.call_count == 4
+    assert dl_mock.call_args[1]['timerange'].starttype == 'date'
+
+    assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
+
+
+def test_download_data_no_markets(mocker, default_conf, caplog):
+    dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    ex = get_patched_exchange(mocker, default_conf)
+    timerange = TimeRange.parse_timerange("20190101-20190102")
+    unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
+                                             timeframes=["1m", "5m"],
+                                             dl_path=make_testdata_path(None),
+                                             timerange=timerange, erase=False
+                                             )
+
+    assert dl_mock.call_count == 0
+    assert "ETH/BTC" in unav_pairs
+    assert "XRP/BTC" in unav_pairs
+    assert log_has("Skipping pair ETH/BTC...", caplog)

freqtrade/tests/exchange/test_binance.py

@@ -0,0 +1,90 @@
+from random import randint
+from unittest.mock import MagicMock
+
+import ccxt
+import pytest
+
+from freqtrade import DependencyException, OperationalException, TemporaryError
+from freqtrade.tests.conftest import get_patched_exchange
+
+
+def test_stoploss_limit_order(default_conf, mocker):
+    api_mock = MagicMock()
+    order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6))
+    order_type = 'stop_loss_limit'
+
+    api_mock.create_order = MagicMock(return_value={
+        'id': order_id,
+        'info': {
+            'foo': 'bar'
+        }
+    })
+
+    default_conf['dry_run'] = False
+    mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y)
+    mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y)
+
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
+
+    with pytest.raises(OperationalException):
+        order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200)
+
+    api_mock.create_order.reset_mock()
+
+    order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
+
+    assert 'id' in order
+    assert 'info' in order
+    assert order['id'] == order_id
+    assert api_mock.create_order.call_args[0][0] == 'ETH/BTC'
+    assert api_mock.create_order.call_args[0][1] == order_type
+    assert api_mock.create_order.call_args[0][2] == 'sell'
+    assert api_mock.create_order.call_args[0][3] == 1
+    assert api_mock.create_order.call_args[0][4] == 200
+    assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220}
+
+    # test exception handling
+    with pytest.raises(DependencyException):
+        api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
+        exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
+
+    with pytest.raises(DependencyException):
+        api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
+        exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
+
+    with pytest.raises(TemporaryError):
+        api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
+        exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
+
+    with pytest.raises(OperationalException, match=r".*DeadBeef.*"):
+        api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
+        exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
+
+
+def test_stoploss_limit_order_dry_run(default_conf, mocker):
+    api_mock = MagicMock()
+    order_type = 'stop_loss_limit'
+    default_conf['dry_run'] = True
+    mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y)
+    mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y)
+
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
+
+    with pytest.raises(OperationalException):
+        order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200)
+
+    api_mock.create_order.reset_mock()
+
+    order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
+
+    assert 'id' in order
+    assert 'info' in order
+    assert 'type' in order
+
+    assert order['type'] == order_type
+    assert order['price'] == 220
+    assert order['amount'] == 1

freqtrade/tests/exchange/test_exchange.py

@@ -101,18 +101,21 @@ def test_destroy(default_conf, mocker, caplog):
 def test_init_exception(default_conf, mocker):
     default_conf['exchange']['name'] = 'wrong_exchange_name'
 
-    with pytest.raises(
-            OperationalException,
-            match='Exchange {} is not supported'.format(default_conf['exchange']['name'])):
+    with pytest.raises(OperationalException,
+                       match=f"Exchange {default_conf['exchange']['name']} is not supported"):
         Exchange(default_conf)
 
     default_conf['exchange']['name'] = 'binance'
-    with pytest.raises(
-            OperationalException,
-            match='Exchange {} is not supported'.format(default_conf['exchange']['name'])):
+    with pytest.raises(OperationalException,
+                       match=f"Exchange {default_conf['exchange']['name']} is not supported"):
         mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError))
         Exchange(default_conf)
 
+    with pytest.raises(OperationalException,
+                       match=r"Initialization of ccxt failed. Reason: DeadBeef"):
+        mocker.patch("ccxt.binance", MagicMock(side_effect=ccxt.BaseError("DeadBeef")))
+        Exchange(default_conf)
+
 
 def test_exchange_resolver(default_conf, mocker, caplog):
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock()))
@@ -1436,87 +1439,11 @@ def test_get_fee(default_conf, mocker, exchange_name):
                            'get_fee', 'calculate_fee')
 
 
-def test_stoploss_limit_order(default_conf, mocker):
-    api_mock = MagicMock()
-    order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6))
-    order_type = 'stop_loss_limit'
-
-    api_mock.create_order = MagicMock(return_value={
-        'id': order_id,
-        'info': {
-            'foo': 'bar'
-        }
-    })
-
-    default_conf['dry_run'] = False
-    mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y)
-
-    exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
-
-    with pytest.raises(OperationalException):
-        order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200)
-
-    api_mock.create_order.reset_mock()
-
-    order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
-
-    assert 'id' in order
-    assert 'info' in order
-    assert order['id'] == order_id
-    assert api_mock.create_order.call_args[0][0] == 'ETH/BTC'
-    assert api_mock.create_order.call_args[0][1] == order_type
-    assert api_mock.create_order.call_args[0][2] == 'sell'
-    assert api_mock.create_order.call_args[0][3] == 1
-    assert api_mock.create_order.call_args[0][4] == 200
-    assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220}
-
-    # test exception handling
-    with pytest.raises(DependencyException):
-        api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance"))
-        exchange = get_patched_exchange(mocker, default_conf, api_mock)
+def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker):
+    exchange = get_patched_exchange(mocker, default_conf, 'bittrex')
+    with pytest.raises(OperationalException, match=r"stoploss_limit is not implemented .*"):
         exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
 
-    with pytest.raises(DependencyException):
-        api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
-        exchange = get_patched_exchange(mocker, default_conf, api_mock)
-        exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
-
-    with pytest.raises(TemporaryError):
-        api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection"))
-        exchange = get_patched_exchange(mocker, default_conf, api_mock)
-        exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
-
-    with pytest.raises(OperationalException):
-        api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef"))
-        exchange = get_patched_exchange(mocker, default_conf, api_mock)
-        exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
-
-
-def test_stoploss_limit_order_dry_run(default_conf, mocker):
-    api_mock = MagicMock()
-    order_type = 'stop_loss_limit'
-    default_conf['dry_run'] = True
-    mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y)
-
-    exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
-
-    with pytest.raises(OperationalException):
-        order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200)
-
-    api_mock.create_order.reset_mock()
-
-    order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200)
-
-    assert 'id' in order
-    assert 'info' in order
-    assert 'type' in order
-
-    assert order['type'] == order_type
-    assert order['price'] == 220
-    assert order['amount'] == 1
-
 
 def test_merge_ft_has_dict(default_conf, mocker):
     mocker.patch.multiple('freqtrade.exchange.Exchange',

freqtrade/tests/optimize/test_hyperopt.py

@@ -381,7 +381,7 @@ def test_save_trials_saves_trials(mocker, hyperopt, caplog) -> None:
     hyperopt.save_trials()
 
     trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle')
-    assert log_has('Saving 1 evaluations to \'{}\''.format(trials_file), caplog)
+    assert log_has("Saving 1 evaluations to '{}'".format(trials_file), caplog)
     mock_dump.assert_called_once()
 
 
@@ -390,7 +390,7 @@ def test_read_trials_returns_trials_file(mocker, hyperopt, caplog) -> None:
     mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials)
     hyperopt_trial = hyperopt.read_trials()
     trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle')
-    assert log_has('Reading Trials from \'{}\''.format(trials_file), caplog)
+    assert log_has("Reading Trials from '{}'".format(trials_file), caplog)
     assert hyperopt_trial == trials
     mock_load.assert_called_once()
 

freqtrade/tests/test_configuration.py

@@ -41,14 +41,14 @@ def test_load_config_invalid_pair(default_conf) -> None:
 def test_load_config_missing_attributes(default_conf) -> None:
     default_conf.pop('exchange')
 
-    with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'):
+    with pytest.raises(ValidationError, match=r".*'exchange' is a required property.*"):
         validate_config_schema(default_conf)
 
 
 def test_load_config_incorrect_stake_amount(default_conf) -> None:
     default_conf['stake_amount'] = 'fake'
 
-    with pytest.raises(ValidationError, match=r'.*\'fake\' does not match \'unlimited\'.*'):
+    with pytest.raises(ValidationError, match=r".*'fake' does not match 'unlimited'.*"):
         validate_config_schema(default_conf)
 
 
@@ -472,7 +472,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None:
 
     assert 'spaces' in config
     assert config['spaces'] == ['all']
-    assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog)
+    assert log_has("Parameter -s/--spaces detected: ['all']", caplog)
     assert "runmode" in config
     assert config['runmode'] == RunMode.HYPEROPT
 
@@ -722,7 +722,7 @@ def test_load_config_default_exchange(all_conf) -> None:
     assert 'exchange' not in all_conf
 
     with pytest.raises(ValidationError,
-                       match=r'\'exchange\' is a required property'):
+                       match=r"'exchange' is a required property"):
         validate_config_schema(all_conf)
 
 
@@ -736,7 +736,7 @@ def test_load_config_default_exchange_name(all_conf) -> None:
     assert 'name' not in all_conf['exchange']
 
     with pytest.raises(ValidationError,
-                       match=r'\'name\' is a required property'):
+                       match=r"'name' is a required property"):
         validate_config_schema(all_conf)
 
 

freqtrade/tests/test_freqtradebot.py

@@ -2414,7 +2414,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf,
 
     mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y)
     mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit)
+    mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit)
 
     freqtrade = FreqtradeBot(default_conf)
     freqtrade.strategy.order_types['stoploss_on_exchange'] = True
@@ -2454,7 +2454,6 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf,
     freqtrade.process_maybe_execute_sell(trade)
     assert trade.stoploss_order_id is None
     assert trade.is_open is False
-    print(trade.sell_reason)
     assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value
     assert rpc_mock.call_count == 2
 

freqtrade/tests/test_utils.py

@@ -1,5 +1,4 @@
 import re
-from pathlib import Path
 from unittest.mock import MagicMock, PropertyMock
 
 import pytest
@@ -70,74 +69,8 @@ def test_create_datadir(caplog, mocker):
     assert len(caplog.record_tuples) == 0
 
 
-def test_download_data(mocker, markets, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
-    patch_exchange(mocker)
-    mocker.patch(
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    mocker.patch.object(Path, "unlink", MagicMock())
-
-    args = [
-        "download-data",
-        "--exchange", "binance",
-        "--pairs", "ETH/BTC", "XRP/BTC",
-        "--erase",
-    ]
-    start_download_data(get_args(args))
-
-    assert dl_mock.call_count == 4
-    assert dl_mock.call_args[1]['timerange'].starttype is None
-    assert dl_mock.call_args[1]['timerange'].stoptype is None
-    assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog)
-    assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
-
-
-def test_download_data_days(mocker, markets, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
-    patch_exchange(mocker)
-    mocker.patch(
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    mocker.patch.object(Path, "unlink", MagicMock())
-
-    args = [
-        "download-data",
-        "--exchange", "binance",
-        "--pairs", "ETH/BTC", "XRP/BTC",
-        "--days", "20",
-    ]
-
-    start_download_data(get_args(args))
-
-    assert dl_mock.call_count == 4
-    assert dl_mock.call_args[1]['timerange'].starttype == 'date'
-
-    assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
-
-
-def test_download_data_no_markets(mocker, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
-    patch_exchange(mocker)
-    mocker.patch(
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
-    )
-    args = [
-        "download-data",
-        "--exchange", "binance",
-        "--pairs", "ETH/BTC", "XRP/BTC",
-    ]
-    start_download_data(get_args(args))
-
-    assert dl_mock.call_count == 0
-    assert log_has("Skipping pair ETH/BTC...", caplog)
-    assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)
-
-
 def test_download_data_keyboardInterrupt(mocker, caplog, markets):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history',
+    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
                            MagicMock(side_effect=KeyboardInterrupt))
     patch_exchange(mocker)
     mocker.patch(
@@ -152,3 +85,21 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets):
         start_download_data(get_args(args))
 
     assert dl_mock.call_count == 1
+
+
+def test_download_data_no_markets(mocker, caplog):
+    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
+                           MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    args = [
+        "download-data",
+        "--exchange", "binance",
+        "--pairs", "ETH/BTC", "XRP/BTC",
+        "--days", "20"
+    ]
+    start_download_data(get_args(args))
+    assert dl_mock.call_args[1]['timerange'].starttype == "date"
+    assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)

freqtrade/utils.py

@@ -2,13 +2,13 @@ import logging
 import sys
 from argparse import Namespace
 from pathlib import Path
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import arrow
 
 from freqtrade.configuration import Configuration, TimeRange
 from freqtrade.configuration.directory_operations import create_userdata_dir
-from freqtrade.data.history import download_pair_history
+from freqtrade.data.history import refresh_backtest_ohlcv_data
 from freqtrade.exchange import available_exchanges
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.state import RunMode
@@ -75,39 +75,23 @@ def start_download_data(args: Namespace) -> None:
     logger.info(f'About to download pairs: {config["pairs"]}, '
                 f'intervals: {config["timeframes"]} to {dl_path}')
 
-    pairs_not_available = []
+    pairs_not_available: List[str] = []
 
     try:
         # Init exchange
         exchange = ExchangeResolver(config['exchange']['name'], config).exchange
 
-        for pair in config["pairs"]:
-            if pair not in exchange.markets:
-                pairs_not_available.append(pair)
-                logger.info(f"Skipping pair {pair}...")
-                continue
-            for ticker_interval in config["timeframes"]:
-                pair_print = pair.replace('/', '_')
-                filename = f'{pair_print}-{ticker_interval}.json'
-                dl_file = dl_path.joinpath(filename)
-                if config.get("erase") and dl_file.exists():
-                    logger.info(
-                        f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
-                    dl_file.unlink()
-
-                logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
-                download_pair_history(datadir=dl_path, exchange=exchange,
-                                      pair=pair, ticker_interval=str(ticker_interval),
-                                      timerange=timerange)
+        pairs_not_available = refresh_backtest_ohlcv_data(
+            exchange, pairs=config["pairs"], timeframes=config["timeframes"],
+            dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
 
     except KeyboardInterrupt:
         sys.exit("SIGINT received, aborting ...")
 
     finally:
         if pairs_not_available:
-            logger.info(
-                f"Pairs [{','.join(pairs_not_available)}] not available "
-                f"on exchange {config['exchange']['name']}.")
+            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
+                        f"on exchange {config['exchange']['name']}.")
 
     # configuration.resolve_pairs_list()
     print(config)

requirements-common.txt

@@ -1,6 +1,6 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.18.1084
+ccxt==1.18.1085
 SQLAlchemy==1.3.7
 python-telegram-bot==11.1.0
 arrow==0.14.5

requirements-dev.txt

@@ -7,7 +7,7 @@ flake8==3.7.8
 flake8-type-annotations==0.1.0
 flake8-tidy-imports==2.0.0
 mypy==0.720
-pytest==5.1.0
+pytest==5.1.1
 pytest-asyncio==0.10.0
 pytest-cov==2.7.1
 pytest-mock==1.10.4

requirements.txt

@@ -2,5 +2,5 @@
 -r requirements-common.txt
 
 numpy==1.17.0
-pandas==0.25.0
+pandas==0.25.1
 scipy==1.3.1