From a0c79bd7272754024ce2b796e94cf45530c119c3 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 11:47:27 +0300 Subject: [PATCH 1/8] make --pairs-file required --- freqtrade/arguments.py | 1 + scripts/download_backtest_data.py | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 7e895177a..04d94a572 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -291,6 +291,7 @@ class Arguments(object): '--pairs-file', help='File containing a list of pairs to download', dest='pairs_file', + required=True, default=None ) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 1c73eae03..85debe703 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -15,12 +15,9 @@ arguments.testdata_dl_options() args = arguments.parse_args() TICKER_INTERVALS = ['1m', '5m'] -PAIRS = [] -if args.pairs_file: - with open(args.pairs_file) as file: - PAIRS = json.load(file) -PAIRS = list(set(PAIRS)) +with open(args.pairs_file) as file: + PAIRS = list(set(json.load(file))) dl_path = DEFAULT_DL_PATH if args.export and os.path.exists(args.export): From e10279b7b4af2dd2b9a0dea28bf668ecbce986e8 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 11:50:33 +0300 Subject: [PATCH 2/8] show default exchange in download_backtest_data.py --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 04d94a572..bf2ba4d6d 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -310,7 +310,7 @@ class Arguments(object): self.parser.add_argument( '--exchange', - help='Exchange name', + help='Exchange name (default: %(default)s)', dest='exchange', type=str, default='bittrex') From 6891054b8470738a00ac2d79a009812d5e1e580f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 12:20:17 +0300 Subject: [PATCH 3/8] use folder user_data/data/exchangename by default and pick pairs.json from that folder by default --- freqtrade/arguments.py | 1 - scripts/download_backtest_data.py | 16 ++++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index bf2ba4d6d..fd4a88826 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -291,7 +291,6 @@ class Arguments(object): '--pairs-file', help='File containing a list of pairs to download', dest='pairs_file', - required=True, default=None ) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 85debe703..c66103769 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -8,7 +8,7 @@ import arrow from freqtrade import (exchange, arguments, misc) -DEFAULT_DL_PATH = 'freqtrade/tests/testdata' +DEFAULT_DL_PATH = 'user_data/data' arguments = arguments.Arguments(sys.argv[1:], 'download utility') arguments.testdata_dl_options() @@ -16,12 +16,16 @@ args = arguments.parse_args() TICKER_INTERVALS = ['1m', '5m'] -with open(args.pairs_file) as file: - PAIRS = list(set(json.load(file))) +dl_path = args.export if args.export and os.path.exists(args.export) else os.path.join(DEFAULT_DL_PATH, args.exchange) +if not os.path.isdir(dl_path): + sys.exit(f'Directory {dl_path} does not exist.') -dl_path = DEFAULT_DL_PATH -if args.export and os.path.exists(args.export): - dl_path = args.export +pairs_file = args.pairs_file if args.pairs_file else os.path.join(dl_path, 'pairs.json') +if not os.path.isfile(pairs_file): + sys.exit(f'No pairs file found with path {pairs_file}.') + +with open(pairs_file) as file: + PAIRS = list(set(json.load(file))) since_time = None if args.days: From d4b431a3350e0d1313b3ffc993ab2c88ade1de61 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 12:30:21 +0300 Subject: [PATCH 4/8] update documentation about download_backtesting_data.py script --- docs/backtesting.md | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index f843d87e0..ab47923c6 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -93,22 +93,30 @@ The full timerange specification: `--timerange=1527595200-1527618600` -**Update testdata directory** -To update your testdata directory, or download into another testdata directory: -```bash -mkdir -p user_data/data/testdata-20180113 -cp freqtrade/tests/testdata/pairs.json user_data/data/testdata-20180113 -cd user_data/data/testdata-20180113 -``` +**Downloading new set of ticker data** +To download new set of backtesting ticker data, you can use a download script. -Possibly edit `pairs.json` file to include/exclude pairs +If you are using Binance for example: +- create a folder `user_data/data/binance` and copy `pairs.json` in that folder. +- update the `pairs.json` to contain the currency pairs you are interested in. ```bash -python3 freqtrade/tests/testdata/download_backtest_data.py -p pairs.json +mkdir -p user_data/data/binance +cp freqtrade/tests/testdata/pairs.json user_data/data/binance ``` -The script will read your `pairs.json` file, and download ticker data -into the current working directory. +Then run: + +```bash +python scripts/download_backtest_data --exchange binance +``` + +This will download ticker data for all the currency pairs you defined in `pairs.json`. + +- To use a different folder than the exchange specific default, use `--export user_data/data/some_directory`. +- To change the exchange used to download the tickers, use `--exchange`. Default is `bittrex`. +- To use `pairs.json` from some other folder, use `--pairs-file some_other_dir/pairs.json`. +- To download ticker data for only 10 days, use `--days 10`. For help about backtesting usage, please refer to From 5c7899ae9864e07efb5817dbd4bc56038fa4dc0d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 12:45:23 +0300 Subject: [PATCH 5/8] flake8 fix --- scripts/download_backtest_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index c66103769..cd64f36bb 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -16,7 +16,8 @@ args = arguments.parse_args() TICKER_INTERVALS = ['1m', '5m'] -dl_path = args.export if args.export and os.path.exists(args.export) else os.path.join(DEFAULT_DL_PATH, args.exchange) +dl_path = args.export if args.export and os.path.exists(args.export) \ + else os.path.join(DEFAULT_DL_PATH, args.exchange) if not os.path.isdir(dl_path): sys.exit(f'Directory {dl_path} does not exist.') From af1ba1e191e42c00c7bec80a74cd06ebcdd756d3 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 12:58:35 +0300 Subject: [PATCH 6/8] split ugly ternary to regular if --- scripts/download_backtest_data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index cd64f36bb..18a543c4e 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -16,8 +16,10 @@ args = arguments.parse_args() TICKER_INTERVALS = ['1m', '5m'] -dl_path = args.export if args.export and os.path.exists(args.export) \ - else os.path.join(DEFAULT_DL_PATH, args.exchange) +dl_path = os.path.join(DEFAULT_DL_PATH, args.exchange) +if args.export: + dl_path = args.export + if not os.path.isdir(dl_path): sys.exit(f'Directory {dl_path} does not exist.') From 639b6bc4f6156e04907bc70f182490887c56d88e Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 14:13:19 +0300 Subject: [PATCH 7/8] set and create default datadir based on used exchange --- freqtrade/arguments.py | 5 ++--- freqtrade/configuration.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index fd4a88826..9e88d08ca 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -4,7 +4,6 @@ This module contains the argument manager class import argparse import logging -import os import re import arrow from typing import List, Tuple, Optional @@ -72,9 +71,9 @@ class Arguments(object): ) self.parser.add_argument( '-d', '--datadir', - help='path to backtest data (default: %(default)s', + help='path to backtest data', dest='datadir', - default=os.path.join('freqtrade', 'tests', 'testdata'), + default=None, type=str, metavar='PATH', ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 77b5b4447..54146199d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -1,7 +1,7 @@ """ This module contains the configuration class """ - +import os import json import logging from argparse import Namespace @@ -113,6 +113,14 @@ class Configuration(object): return config + def _create_default_datadir(self, config: Dict[str, Any]) -> str: + exchange_name = config.get('exchange', {}).get('name').lower() + default_path = os.path.join('user_data', 'data', exchange_name) + if not os.path.isdir(default_path): + os.makedirs(default_path) + logger.info(f'Created data directory: {default_path}') + return default_path + def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Extract information for sys.argv and load Backtesting configuration @@ -145,7 +153,9 @@ class Configuration(object): # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self.args.datadir}) - logger.info('Using data folder: %s ...', self.args.datadir) + else: + config.update({'datadir': self._create_default_datadir(config)}) + logger.info('Using data folder: %s ...', config.get('datadir')) # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: From 3321e4cafd0761acd82e8fc9fd96c2e8d2ff9442 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 14:14:04 +0300 Subject: [PATCH 8/8] travis should run hyperopt and backtesting using tests/testdata tickers --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1cff5c04b..c3c118654 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ jobs: - script: pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - script: - cp config.json.example config.json - - python freqtrade/main.py backtesting + - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting - script: - cp config.json.example config.json - - python freqtrade/main.py hyperopt -e 5 + - python freqtrade/main.py --datadir freqtrade/tests/testdata hyperopt -e 5 - script: flake8 freqtrade - script: mypy freqtrade after_success: