diff --git a/.travis.yml b/.travis.yml index 1cff5c04b..c3c118654 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ jobs: - script: pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - script: - cp config.json.example config.json - - python freqtrade/main.py backtesting + - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting - script: - cp config.json.example config.json - - python freqtrade/main.py hyperopt -e 5 + - python freqtrade/main.py --datadir freqtrade/tests/testdata hyperopt -e 5 - script: flake8 freqtrade - script: mypy freqtrade after_success: diff --git a/docs/backtesting.md b/docs/backtesting.md index f843d87e0..ab47923c6 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -93,22 +93,30 @@ The full timerange specification: `--timerange=1527595200-1527618600` -**Update testdata directory** -To update your testdata directory, or download into another testdata directory: -```bash -mkdir -p user_data/data/testdata-20180113 -cp freqtrade/tests/testdata/pairs.json user_data/data/testdata-20180113 -cd user_data/data/testdata-20180113 -``` +**Downloading new set of ticker data** +To download new set of backtesting ticker data, you can use a download script. -Possibly edit `pairs.json` file to include/exclude pairs +If you are using Binance for example: +- create a folder `user_data/data/binance` and copy `pairs.json` in that folder. +- update the `pairs.json` to contain the currency pairs you are interested in. ```bash -python3 freqtrade/tests/testdata/download_backtest_data.py -p pairs.json +mkdir -p user_data/data/binance +cp freqtrade/tests/testdata/pairs.json user_data/data/binance ``` -The script will read your `pairs.json` file, and download ticker data -into the current working directory. +Then run: + +```bash +python scripts/download_backtest_data --exchange binance +``` + +This will download ticker data for all the currency pairs you defined in `pairs.json`. + +- To use a different folder than the exchange specific default, use `--export user_data/data/some_directory`. +- To change the exchange used to download the tickers, use `--exchange`. Default is `bittrex`. +- To use `pairs.json` from some other folder, use `--pairs-file some_other_dir/pairs.json`. +- To download ticker data for only 10 days, use `--days 10`. For help about backtesting usage, please refer to diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 7e895177a..9e88d08ca 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -4,7 +4,6 @@ This module contains the argument manager class import argparse import logging -import os import re import arrow from typing import List, Tuple, Optional @@ -72,9 +71,9 @@ class Arguments(object): ) self.parser.add_argument( '-d', '--datadir', - help='path to backtest data (default: %(default)s', + help='path to backtest data', dest='datadir', - default=os.path.join('freqtrade', 'tests', 'testdata'), + default=None, type=str, metavar='PATH', ) @@ -309,7 +308,7 @@ class Arguments(object): self.parser.add_argument( '--exchange', - help='Exchange name', + help='Exchange name (default: %(default)s)', dest='exchange', type=str, default='bittrex') diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 77b5b4447..54146199d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -1,7 +1,7 @@ """ This module contains the configuration class """ - +import os import json import logging from argparse import Namespace @@ -113,6 +113,14 @@ class Configuration(object): return config + def _create_default_datadir(self, config: Dict[str, Any]) -> str: + exchange_name = config.get('exchange', {}).get('name').lower() + default_path = os.path.join('user_data', 'data', exchange_name) + if not os.path.isdir(default_path): + os.makedirs(default_path) + logger.info(f'Created data directory: {default_path}') + return default_path + def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Extract information for sys.argv and load Backtesting configuration @@ -145,7 +153,9 @@ class Configuration(object): # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self.args.datadir}) - logger.info('Using data folder: %s ...', self.args.datadir) + else: + config.update({'datadir': self._create_default_datadir(config)}) + logger.info('Using data folder: %s ...', config.get('datadir')) # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 1c73eae03..18a543c4e 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -8,24 +8,28 @@ import arrow from freqtrade import (exchange, arguments, misc) -DEFAULT_DL_PATH = 'freqtrade/tests/testdata' +DEFAULT_DL_PATH = 'user_data/data' arguments = arguments.Arguments(sys.argv[1:], 'download utility') arguments.testdata_dl_options() args = arguments.parse_args() TICKER_INTERVALS = ['1m', '5m'] -PAIRS = [] -if args.pairs_file: - with open(args.pairs_file) as file: - PAIRS = json.load(file) -PAIRS = list(set(PAIRS)) - -dl_path = DEFAULT_DL_PATH -if args.export and os.path.exists(args.export): +dl_path = os.path.join(DEFAULT_DL_PATH, args.exchange) +if args.export: dl_path = args.export +if not os.path.isdir(dl_path): + sys.exit(f'Directory {dl_path} does not exist.') + +pairs_file = args.pairs_file if args.pairs_file else os.path.join(dl_path, 'pairs.json') +if not os.path.isfile(pairs_file): + sys.exit(f'No pairs file found with path {pairs_file}.') + +with open(pairs_file) as file: + PAIRS = list(set(json.load(file))) + since_time = None if args.days: since_time = arrow.utcnow().shift(days=-args.days).timestamp * 1000