diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py deleted file mode 100755 index 96e0cbce8..000000000 --- a/scripts/convert_backtestdata.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -""" -Script to display when the bot will buy a specific pair - -Mandatory Cli parameters: --p / --pair: pair to examine - -Optional Cli parameters --d / --datadir: path to pair backtest data ---timerange: specify what timerange of data to use. --l / --live: Live, to download the latest ticker for the pair -""" -import logging -import sys -from argparse import Namespace -from os import path -import glob -import json -import re -from typing import List, Dict -import gzip - -from freqtrade.arguments import Arguments -from freqtrade import misc, constants -from pandas import DataFrame - -import dateutil.parser - -logger = logging.getLogger('freqtrade') - - -def load_old_file(filename) -> (List[Dict], bool): - if not path.isfile(filename): - logger.warning("filename %s does not exist", filename) - return (None, False) - logger.debug('Loading ticker data from file %s', filename) - - pairdata = None - - if filename.endswith('.gz'): - logger.debug('Loading ticker data from file %s', filename) - is_zip = True - with gzip.open(filename) as tickerdata: - pairdata = json.load(tickerdata) - else: - is_zip = False - with open(filename) as tickerdata: - pairdata = json.load(tickerdata) - return (pairdata, is_zip) - - -def parse_old_backtest_data(ticker) -> DataFrame: - """ - Reads old backtest data - Format: "O": 8.794e-05, - "H": 8.948e-05, - "L": 8.794e-05, - "C": 8.88e-05, - "V": 991.09056638, - "T": "2017-11-26T08:50:00", - "BV": 0.0877869 - """ - - columns = {'C': 'close', 'V': 'volume', 'O': 'open', - 'H': 'high', 'L': 'low', 'T': 'date'} - - frame = DataFrame(ticker) \ - .rename(columns=columns) - if 'BV' in frame: - frame.drop('BV', 1, inplace=True) - if 'date' not in frame: - logger.warning("Date not in frame - probably not a Ticker file") - return None - frame.sort_values('date', inplace=True) - return frame - - -def convert_dataframe(frame: DataFrame): - """Convert dataframe to new format""" - # reorder columns: - cols = ['date', 'open', 'high', 'low', 'close', 'volume'] - frame = frame[cols] - - # Make sure parsing/printing data is assumed to be UTC - frame['date'] = frame['date'].apply( - lambda d: int(dateutil.parser.parse(d+'+00:00').timestamp()) * 1000) - frame['date'] = frame['date'].astype('int64') - # Convert columns one by one to preserve type. - by_column = [frame[x].values.tolist() for x in frame.columns] - return list(list(x) for x in zip(*by_column)) - - -def convert_file(filename: str, filename_new: str) -> None: - """Converts a file from old format to ccxt format""" - (pairdata, is_zip) = load_old_file(filename) - if pairdata and type(pairdata) is list: - if type(pairdata[0]) is list: - logger.error("pairdata for %s already in new format", filename) - return - - frame = parse_old_backtest_data(pairdata) - # Convert frame to new format - if frame is not None: - frame1 = convert_dataframe(frame) - misc.file_dump_json(filename_new, frame1, is_zip) - - -def convert_main(args: Namespace) -> None: - """ - converts a folder given in --datadir from old to new format to support ccxt - """ - - workdir = path.join(args.datadir, "") - logger.info("Workdir: %s", workdir) - - for filename in glob.glob(workdir + "*.json"): - # swap currency names - ret = re.search(r'[A-Z_]{7,}', path.basename(filename)) - if args.norename: - filename_new = filename - else: - if not ret: - logger.warning("file %s could not be converted, could not extract currencies", - filename) - continue - pair = ret.group(0) - currencies = pair.split("_") - if len(currencies) != 2: - logger.warning("file %s could not be converted, could not extract currencies", - filename) - continue - - ret_integer = re.search(r'\d+(?=\.json)', path.basename(filename)) - ret_string = re.search(r'(\d+[mhdw])(?=\.json)', path.basename(filename)) - - if ret_integer: - minutes = int(ret_integer.group(0)) - # default to adding 'm' to end of minutes for new interval name - interval = str(minutes) + 'm' - # but check if there is a mapping between int and string also - for str_interval, minutes_interval in constants.TICKER_INTERVAL_MINUTES.items(): - if minutes_interval == minutes: - interval = str_interval - break - # change order on pairs if old ticker interval found - - filename_new = path.join(path.dirname(filename), - f"{currencies[1]}_{currencies[0]}-{interval}.json") - - elif ret_string: - interval = ret_string.group(0) - filename_new = path.join(path.dirname(filename), - f"{currencies[0]}_{currencies[1]}-{interval}.json") - - else: - logger.warning("file %s could not be converted, interval not found", filename) - continue - - logger.debug("Converting and renaming %s to %s", filename, filename_new) - convert_file(filename, filename_new) - - -def convert_parse_args(args: List[str]) -> Namespace: - """ - Parse args passed to the script - :param args: Cli arguments - :return: args: Array with all arguments - """ - arguments = Arguments(args, 'Convert datafiles') - arguments.parser.add_argument( - '-d', '--datadir', - help='path to backtest data (default: %(default)s', - dest='datadir', - default=path.join('freqtrade', 'tests', 'testdata'), - type=str, - metavar='PATH', - ) - arguments.parser.add_argument( - '-n', '--norename', - help='don''t rename files from BTC_ to _BTC - ' - 'Note that not renaming will overwrite source files', - dest='norename', - default=False, - action='store_true' - ) - - return arguments.parse_args() - - -def main(sysargv: List[str]) -> None: - """ - This function will initiate the bot and start the trading loop. - :return: None - """ - logger.info('Starting Dataframe conversation') - convert_main(convert_parse_args(sysargv)) - - -if __name__ == '__main__': - main(sys.argv[1:])