From 60ed4b9d1e86c16c5a911c891c08a419072574f3 Mon Sep 17 00:00:00 2001 From: kryofly Date: Sat, 6 Jan 2018 23:24:35 +0100 Subject: [PATCH 1/2] --datadir argument This argument enables usage of different backtesting directories. Useful if one wants compare backtesting performance over time. --- freqtrade/misc.py | 9 ++++++ freqtrade/optimize/__init__.py | 29 ++++++++++---------- freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/hyperopt.py | 4 +-- freqtrade/tests/optimize/test_backtesting.py | 15 +++++----- freqtrade/tests/optimize/test_optimize.py | 22 +++++++-------- freqtrade/tests/test_dataframe.py | 2 +- 7 files changed, 47 insertions(+), 36 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 900aa8763..2d43fa581 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -126,6 +126,15 @@ def parse_args(args: List[str]): action='store_true', dest='dry_run_db', ) + parser.add_argument( + '-dd', '--datadir', + help='path to backtest data (default freqdata/tests/testdata', + dest='datadir', + default='freqtrade/tests/testdata', + type=str, + metavar='PATH', + ) + build_subcommands(parser) parsed_args = parser.parse_args(args) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index f36c28fbc..2d73c3215 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -12,12 +12,12 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe logger = logging.getLogger(__name__) -def load_tickerdata_file(pair, ticker_interval): +def load_tickerdata_file(datadir, pair, ticker_interval): """ Load a pair from file, :return dict OR empty if unsuccesful """ - path = testdata_path() + path = make_testdata_path(datadir) file = '{abspath}/{pair}-{ticker_interval}.json'.format( abspath=path, pair=pair, @@ -33,7 +33,7 @@ def load_tickerdata_file(pair, ticker_interval): return pairdata -def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None, +def load_data(datadir: str, ticker_interval: int = 5, pairs: Optional[List[str]] = None, refresh_pairs: Optional[bool] = False) -> Dict[str, List]: """ Loads ticker history data for the given parameters @@ -47,16 +47,16 @@ def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None, # If the user force the refresh of pairs if refresh_pairs: - logger.info('Download data for all pairs and store them in freqtrade/tests/testsdata') - download_pairs(_pairs) + logger.info('Download data for all pairs and store them in %s', datadir) + download_pairs(datadir, _pairs) for pair in _pairs: - pairdata = load_tickerdata_file(pair, ticker_interval) + pairdata = load_tickerdata_file(datadir, pair, ticker_interval) if not pairdata: # download the tickerdata from exchange - download_backtesting_testdata(pair=pair, interval=ticker_interval) + download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval) # and retry reading the pair - pairdata = load_tickerdata_file(pair, ticker_interval) + pairdata = load_tickerdata_file(datadir, pair, ticker_interval) result[pair] = pairdata return result @@ -67,17 +67,18 @@ def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: for pair, pair_data in tickerdata.items()} -def testdata_path() -> str: +def make_testdata_path(datadir: str) -> str: """Return the path where testdata files are stored""" - return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'tests', 'testdata')) + return datadir or os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', 'tests', 'testdata')) -def download_pairs(pairs: List[str]) -> bool: +def download_pairs(datadir, pairs: List[str]) -> bool: """For each pairs passed in parameters, download 1 and 5 ticker intervals""" for pair in pairs: try: for interval in [1, 5]: - download_backtesting_testdata(pair=pair, interval=interval) + download_backtesting_testdata(datadir, pair=pair, interval=interval) except BaseException: logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format( pair=pair, @@ -87,7 +88,7 @@ def download_pairs(pairs: List[str]) -> bool: return True -def download_backtesting_testdata(pair: str, interval: int = 5) -> bool: +def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool: """ Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters Based on @Rybolov work: https://github.com/rybolov/freqtrade-data @@ -95,7 +96,7 @@ def download_backtesting_testdata(pair: str, interval: int = 5) -> bool: :return: bool """ - path = testdata_path() + path = make_testdata_path(datadir) logger.info('Download the pair: "{pair}", Interval: {interval} min'.format( pair=pair, interval=interval, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index afcc43367..315f960d8 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -162,7 +162,7 @@ def start(args): data[pair] = exchange.get_ticker_history(pair, args.ticker_interval) else: logger.info('Using local backtesting data (using whitelist in given config) ...') - data = optimize.load_data(pairs=pairs, ticker_interval=args.ticker_interval, + data = optimize.load_data(args.datadir, pairs=pairs, ticker_interval=args.ticker_interval, refresh_pairs=args.refresh_pairs) logger.info('Using stake_currency: %s ...', config['stake_currency']) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d0d0916f8..ad668972c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -36,7 +36,7 @@ CURRENT_BEST_LOSS = 100 EXPECTED_MAX_PROFIT = 3.85 # Configuration and data used by hyperopt -PROCESSED = optimize.preprocess(optimize.load_data()) +PROCESSED = None # optimize.preprocess(optimize.load_data()) OPTIMIZE_CONFIG = hyperopt_optimize_conf() # Monkey patch config @@ -224,7 +224,7 @@ def start(args): config = load_config(args.config) pairs = config['exchange']['pair_whitelist'] PROCESSED = optimize.preprocess(optimize.load_data( - pairs=pairs, ticker_interval=args.ticker_interval)) + args.datadir, pairs=pairs, ticker_interval=args.ticker_interval)) if args.mongodb: logger.info('Using mongodb ...') diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 06623d433..5f899a48a 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -32,7 +32,7 @@ def test_generate_text_table(): def test_get_timeframe(): data = preprocess(optimize.load_data( - ticker_interval=1, pairs=['BTC_UNITEST'])) + None, ticker_interval=1, pairs=['BTC_UNITEST'])) min_date, max_date = get_timeframe(data) assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' assert max_date.isoformat() == '2017-11-14T22:59:00+00:00' @@ -42,7 +42,7 @@ def test_backtest(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) exchange._API = Bittrex({'key': '', 'secret': ''}) - data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) + data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH']) results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) assert not results.empty @@ -53,7 +53,7 @@ def test_backtest_1min_ticker_interval(default_conf, mocker): exchange._API = Bittrex({'key': '', 'secret': ''}) # Run a backtesting for an exiting 5min ticker_interval - data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) + data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST']) results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 1, True) assert not results.empty @@ -67,7 +67,7 @@ def trim_dictlist(dl, num): def load_data_test(what): - data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) + data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST']) data = trim_dictlist(data, -100) pair = data['BTC_UNITEST'] datalen = len(pair) @@ -124,7 +124,7 @@ def simple_backtest(config, contour, num_results): def test_backtest2(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) - data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) + data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH']) results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) assert not results.empty @@ -149,8 +149,8 @@ def test_backtest_pricecontours(default_conf, mocker): simple_backtest(default_conf, contour, numres) -def mocked_load_data(pairs=[], ticker_interval=0, refresh_pairs=False): - tickerdata = optimize.load_tickerdata_file('BTC_UNITEST', 1) +def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False): + tickerdata = optimize.load_tickerdata_file(datadir, 'BTC_UNITEST', 1) pairdata = {'BTC_UNITEST': tickerdata} return trim_dictlist(pairdata, -100) @@ -165,6 +165,7 @@ def test_backtest_start(default_conf, mocker, caplog): args.ticker_interval = 1 args.level = 10 args.live = False + args.datadir = None backtesting.start(args) # check the logs, that will contain the backtest result exists = ['Using max_open_trades: 1 ...', diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 62b798c2c..a5892f278 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -5,7 +5,7 @@ import logging from shutil import copyfile from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex -from freqtrade.optimize.__init__ import testdata_path, download_pairs,\ +from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ download_backtesting_testdata, load_tickerdata_file # Change this if modifying BTC_UNITEST testdatafile @@ -51,7 +51,7 @@ def test_load_data_5min_ticker(default_conf, ticker_history, mocker, caplog): file = 'freqtrade/tests/testdata/BTC_ETH-5.json' _backup_file(file, copy_file=True) - optimize.load_data(pairs=['BTC_ETH']) + optimize.load_data(None, pairs=['BTC_ETH']) assert os.path.isfile(file) is True assert ('freqtrade.optimize', logging.INFO, @@ -68,7 +68,7 @@ def test_load_data_1min_ticker(default_conf, ticker_history, mocker, caplog): file = 'freqtrade/tests/testdata/BTC_ETH-1.json' _backup_file(file, copy_file=True) - optimize.load_data(ticker_interval=1, pairs=['BTC_ETH']) + optimize.load_data(None, ticker_interval=1, pairs=['BTC_ETH']) assert os.path.isfile(file) is True assert ('freqtrade.optimize', logging.INFO, @@ -85,7 +85,7 @@ def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, capl file = 'freqtrade/tests/testdata/BTC_MEME-1.json' _backup_file(file) - optimize.load_data(ticker_interval=1, pairs=['BTC_MEME']) + optimize.load_data(None, ticker_interval=1, pairs=['BTC_MEME']) assert os.path.isfile(file) is True assert ('freqtrade.optimize', logging.INFO, @@ -95,7 +95,7 @@ def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, capl def test_testdata_path(): - assert os.path.join('freqtrade', 'tests', 'testdata') in testdata_path() + assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None) def test_download_pairs(default_conf, ticker_history, mocker): @@ -113,7 +113,7 @@ def test_download_pairs(default_conf, ticker_history, mocker): _backup_file(file2_1) _backup_file(file2_5) - assert download_pairs(pairs=['BTC-MEME', 'BTC-CFI']) is True + assert download_pairs(None, pairs=['BTC-MEME', 'BTC-CFI']) is True assert os.path.isfile(file1_1) is True assert os.path.isfile(file1_5) is True @@ -139,7 +139,7 @@ def test_download_pairs_exception(default_conf, ticker_history, mocker, caplog): _backup_file(file1_1) _backup_file(file1_5) - download_pairs(pairs=['BTC-MEME']) + download_pairs(None, pairs=['BTC-MEME']) # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) @@ -157,7 +157,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): # Download a 1 min ticker file file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json' _backup_file(file1) - download_backtesting_testdata(pair="BTC-XEL", interval=1) + download_backtesting_testdata(None, pair="BTC-XEL", interval=1) assert os.path.isfile(file1) is True _clean_test_file(file1) @@ -165,12 +165,12 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json' _backup_file(file2) - download_backtesting_testdata(pair="BTC-STORJ", interval=5) + download_backtesting_testdata(None, pair="BTC-STORJ", interval=5) assert os.path.isfile(file2) is True _clean_test_file(file2) def test_load_tickerdata_file(): - assert not load_tickerdata_file('BTC_UNITEST', 7) - tickerdata = load_tickerdata_file('BTC_UNITEST', 1) + assert not load_tickerdata_file(None, 'BTC_UNITEST', 7) + tickerdata = load_tickerdata_file(None, 'BTC_UNITEST', 1) assert _btc_unittest_length == len(tickerdata) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index 916985406..cd8af2c52 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -7,7 +7,7 @@ _pairs = ['BTC_ETH'] def load_dataframe_pair(pairs): - ld = freqtrade.optimize.load_data(ticker_interval=5, pairs=pairs) + ld = freqtrade.optimize.load_data(None, ticker_interval=5, pairs=pairs) assert isinstance(ld, dict) assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] From 0c9d862a49e698897443e25aa37dfed9807543aa Mon Sep 17 00:00:00 2001 From: kryofly Date: Sun, 7 Jan 2018 10:15:26 +0100 Subject: [PATCH 2/2] docs: --datadir documentation --- docs/backtesting.md | 7 ++++++- docs/bot-usage.md | 5 ++++- docs/hyperopt.md | 5 +++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index d7073365f..c426e2b5c 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -46,6 +46,11 @@ python3 ./freqtrade/main.py backtesting --realistic-simulation --refresh-pairs-c python3 ./freqtrade/main.py backtesting --realistic-simulation --live ``` +**Using a different on-disk ticker-data source** +```bash +python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101 +``` + For help about backtesting usage, please refer to [Backtesting commands](#backtesting-commands). @@ -101,4 +106,4 @@ strategies, your configuration, and the crypto-currency you have set up. ## Next step Great, your strategy is profitable. What if the bot can give your the optimal parameters to use for your strategy? -Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md) \ No newline at end of file +Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 79044691e..d3dcf8659 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -26,6 +26,9 @@ optional arguments: specify configuration file (default: config.json) -v, --verbose be verbose --version show program's version number and exit + -dd PATH, --datadir PATH + Path is from where backtesting and hyperopt will load the + ticker data files (default freqdata/tests/testdata). --dynamic-whitelist [INT] dynamically generate and update whitelist based on 24h BaseVolume (Default 20 currencies) @@ -133,4 +136,4 @@ in [misc.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/misc.py#L ## Next step The optimal strategy of the bot will change with time depending of the market trends. The next step is to -[optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md). \ No newline at end of file +[optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md). diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 6cf0d3f90..24a9dbc51 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -163,6 +163,11 @@ We strongly recommend to use `screen` to prevent any connection loss. python3 ./freqtrade/main.py -c config.json hyperopt ``` +### Execute hyperopt with different ticker-data source +If you would like to learn parameters using an alternate ticke-data that +you have on-disk, use the --datadir PATH option. Default hyperopt will +use data from directory freqtrade/tests/testdata. + ### Hyperopt with MongoDB Hyperopt with MongoDB, is like Hyperopt under steroids. As you saw by executing the previous command is the execution takes a long time.