mirror of
https://github.com/freqtrade/freqtrade.git
synced 2024-11-10 10:21:59 +00:00
Merge pull request #1436 from freqtrade/remove_convert_dataframe
Remove convert_backtestdata - this is not usefull anymore
This commit is contained in:
Misagh
GitHub
commit
3411d3d5fa
|
|
@ -1,200 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script to display when the bot will buy a specific pair
|
||||
|
||||
Mandatory Cli parameters:
|
||||
-p / --pair: pair to examine
|
||||
|
||||
Optional Cli parameters
|
||||
-d / --datadir: path to pair backtest data
|
||||
--timerange: specify what timerange of data to use.
|
||||
-l / --live: Live, to download the latest ticker for the pair
|
||||
"""
|
||||
import logging
|
||||
import sys
|
||||
from argparse import Namespace
|
||||
from os import path
|
||||
import glob
|
||||
import json
|
||||
import re
|
||||
from typing import List, Dict
|
||||
import gzip
|
||||
|
||||
from freqtrade.arguments import Arguments
|
||||
from freqtrade import misc, constants
|
||||
from pandas import DataFrame
|
||||
|
||||
import dateutil.parser
|
||||
|
||||
logger = logging.getLogger('freqtrade')
|
||||
|
||||
|
||||
def load_old_file(filename) -> (List[Dict], bool):
|
||||
if not path.isfile(filename):
|
||||
logger.warning("filename %s does not exist", filename)
|
||||
return (None, False)
|
||||
logger.debug('Loading ticker data from file %s', filename)
|
||||
|
||||
pairdata = None
|
||||
|
||||
if filename.endswith('.gz'):
|
||||
logger.debug('Loading ticker data from file %s', filename)
|
||||
is_zip = True
|
||||
with gzip.open(filename) as tickerdata:
|
||||
pairdata = json.load(tickerdata)
|
||||
else:
|
||||
is_zip = False
|
||||
with open(filename) as tickerdata:
|
||||
pairdata = json.load(tickerdata)
|
||||
return (pairdata, is_zip)
|
||||
|
||||
|
||||
def parse_old_backtest_data(ticker) -> DataFrame:
|
||||
"""
|
||||
Reads old backtest data
|
||||
Format: "O": 8.794e-05,
|
||||
"H": 8.948e-05,
|
||||
"L": 8.794e-05,
|
||||
"C": 8.88e-05,
|
||||
"V": 991.09056638,
|
||||
"T": "2017-11-26T08:50:00",
|
||||
"BV": 0.0877869
|
||||
"""
|
||||
|
||||
columns = {'C': 'close', 'V': 'volume', 'O': 'open',
|
||||
'H': 'high', 'L': 'low', 'T': 'date'}
|
||||
|
||||
frame = DataFrame(ticker) \
|
||||
.rename(columns=columns)
|
||||
if 'BV' in frame:
|
||||
frame.drop('BV', 1, inplace=True)
|
||||
if 'date' not in frame:
|
||||
logger.warning("Date not in frame - probably not a Ticker file")
|
||||
return None
|
||||
frame.sort_values('date', inplace=True)
|
||||
return frame
|
||||
|
||||
|
||||
def convert_dataframe(frame: DataFrame):
|
||||
"""Convert dataframe to new format"""
|
||||
# reorder columns:
|
||||
cols = ['date', 'open', 'high', 'low', 'close', 'volume']
|
||||
frame = frame[cols]
|
||||
|
||||
# Make sure parsing/printing data is assumed to be UTC
|
||||
frame['date'] = frame['date'].apply(
|
||||
lambda d: int(dateutil.parser.parse(d+'+00:00').timestamp()) * 1000)
|
||||
frame['date'] = frame['date'].astype('int64')
|
||||
# Convert columns one by one to preserve type.
|
||||
by_column = [frame[x].values.tolist() for x in frame.columns]
|
||||
return list(list(x) for x in zip(*by_column))
|
||||
|
||||
|
||||
def convert_file(filename: str, filename_new: str) -> None:
|
||||
"""Converts a file from old format to ccxt format"""
|
||||
(pairdata, is_zip) = load_old_file(filename)
|
||||
if pairdata and type(pairdata) is list:
|
||||
if type(pairdata[0]) is list:
|
||||
logger.error("pairdata for %s already in new format", filename)
|
||||
return
|
||||
|
||||
frame = parse_old_backtest_data(pairdata)
|
||||
# Convert frame to new format
|
||||
if frame is not None:
|
||||
frame1 = convert_dataframe(frame)
|
||||
misc.file_dump_json(filename_new, frame1, is_zip)
|
||||
|
||||
|
||||
def convert_main(args: Namespace) -> None:
|
||||
"""
|
||||
converts a folder given in --datadir from old to new format to support ccxt
|
||||
"""
|
||||
|
||||
workdir = path.join(args.datadir, "")
|
||||
logger.info("Workdir: %s", workdir)
|
||||
|
||||
for filename in glob.glob(workdir + "*.json"):
|
||||
# swap currency names
|
||||
ret = re.search(r'[A-Z_]{7,}', path.basename(filename))
|
||||
if args.norename:
|
||||
filename_new = filename
|
||||
else:
|
||||
if not ret:
|
||||
logger.warning("file %s could not be converted, could not extract currencies",
|
||||
filename)
|
||||
continue
|
||||
pair = ret.group(0)
|
||||
currencies = pair.split("_")
|
||||
if len(currencies) != 2:
|
||||
logger.warning("file %s could not be converted, could not extract currencies",
|
||||
filename)
|
||||
continue
|
||||
|
||||
ret_integer = re.search(r'\d+(?=\.json)', path.basename(filename))
|
||||
ret_string = re.search(r'(\d+[mhdw])(?=\.json)', path.basename(filename))
|
||||
|
||||
if ret_integer:
|
||||
minutes = int(ret_integer.group(0))
|
||||
# default to adding 'm' to end of minutes for new interval name
|
||||
interval = str(minutes) + 'm'
|
||||
# but check if there is a mapping between int and string also
|
||||
for str_interval, minutes_interval in constants.TICKER_INTERVAL_MINUTES.items():
|
||||
if minutes_interval == minutes:
|
||||
interval = str_interval
|
||||
break
|
||||
# change order on pairs if old ticker interval found
|
||||
|
||||
filename_new = path.join(path.dirname(filename),
|
||||
f"{currencies[1]}_{currencies[0]}-{interval}.json")
|
||||
|
||||
elif ret_string:
|
||||
interval = ret_string.group(0)
|
||||
filename_new = path.join(path.dirname(filename),
|
||||
f"{currencies[0]}_{currencies[1]}-{interval}.json")
|
||||
|
||||
else:
|
||||
logger.warning("file %s could not be converted, interval not found", filename)
|
||||
continue
|
||||
|
||||
logger.debug("Converting and renaming %s to %s", filename, filename_new)
|
||||
convert_file(filename, filename_new)
|
||||
|
||||
|
||||
def convert_parse_args(args: List[str]) -> Namespace:
|
||||
"""
|
||||
Parse args passed to the script
|
||||
:param args: Cli arguments
|
||||
:return: args: Array with all arguments
|
||||
"""
|
||||
arguments = Arguments(args, 'Convert datafiles')
|
||||
arguments.parser.add_argument(
|
||||
'-d', '--datadir',
|
||||
help='path to backtest data (default: %(default)s',
|
||||
dest='datadir',
|
||||
default=path.join('freqtrade', 'tests', 'testdata'),
|
||||
type=str,
|
||||
metavar='PATH',
|
||||
)
|
||||
arguments.parser.add_argument(
|
||||
'-n', '--norename',
|
||||
help='don''t rename files from BTC_<PAIR> to <PAIR>_BTC - '
|
||||
'Note that not renaming will overwrite source files',
|
||||
dest='norename',
|
||||
default=False,
|
||||
action='store_true'
|
||||
)
|
||||
|
||||
return arguments.parse_args()
|
||||
|
||||
|
||||
def main(sysargv: List[str]) -> None:
|
||||
"""
|
||||
This function will initiate the bot and start the trading loop.
|
||||
:return: None
|
||||
"""
|
||||
logger.info('Starting Dataframe conversation')
|
||||
convert_main(convert_parse_args(sysargv))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[1:])
|
||||
Loading…
Reference in New Issue
Block a user