Keep in misc file only tool functions

freqtrade/misc.py

@@ -1,30 +1,26 @@
-import argparse
+"""
-import enum
+Various tool function for Freqtrade and scripts
+"""
+
+import re
 import json
 import logging
-import time
-import os
-import re
 from datetime import datetime
-from typing import Any, Callable, Dict, List
-
 import numpy as np
-from jsonschema import Draft4Validator, validate
-from jsonschema.exceptions import ValidationError, best_match
-from wrapt import synchronized
-
-from freqtrade import __version__
 
 logger = logging.getLogger(__name__)
 
 
-class State(enum.Enum):
+def shorten_date(_date):
-    RUNNING = 0
+    """
-    STOPPED = 1
+    Trim the date so it fits on small screens
-
+    """
-
+    new_date = re.sub('seconds?', 'sec', _date)
-# Current application state
+    new_date = re.sub('minutes?', 'min', new_date)
-_STATE = State.STOPPED
+    new_date = re.sub('hours?', 'h', new_date)
+    new_date = re.sub('days?', 'd', new_date)
+    new_date = re.sub('^an?', '1', new_date)
+    return new_date
 
 
 ############################################
@@ -32,7 +28,6 @@ _STATE = State.STOPPED
 # Matplotlib doesn't support ::datetime64, #
 # so we need to convert it into ::datetime #
 ############################################
-
 def datesarray_to_datetimearray(dates):
     """
     Convert an pandas-array of timestamps into
@@ -41,13 +36,18 @@ def datesarray_to_datetimearray(dates):
     """
     times = []
     dates = dates.astype(datetime)
-    for i in range(0, dates.size):
+    for index in range(0, dates.size):
-        date = dates[i].to_pydatetime()
+        date = dates[index].to_pydatetime()
         times.append(date)
     return np.array(times)
 
 
 def common_datearray(dfs):
+    """
+    Return dates from Dataframe
+    :param dfs: Dataframe
+    :return: List of dates
+    """
     alldates = {}
     for pair, pair_data in dfs.items():
         dates = datesarray_to_datetimearray(pair_data['date'])
@@ -61,375 +61,11 @@ def common_datearray(dfs):
 
 
 def file_dump_json(filename, data) -> None:
-    with open(filename, 'w') as fp:
-        json.dump(data, fp)
-
-
-@synchronized
-def update_state(state: State) -> None:
     """
-    Updates the application state
+    Dump JSON data into a file
-    :param state: new state
+    :param filename: file to create
-    :return: None
+    :param data: JSON Data to save
-    """
-    global _STATE
-    _STATE = state
-
-
-@synchronized
-def get_state() -> State:
-    """
-    Gets the current application state
     :return:
     """
-    return _STATE
+    with open(filename, 'w') as file:
-
+        json.dump(data, file)
-
-def load_config(path: str) -> Dict:
-    """
-    Loads a config file from the given path
-    :param path: path as str
-    :return: configuration as dictionary
-    """
-    with open(path) as file:
-        conf = json.load(file)
-    if 'internals' not in conf:
-        conf['internals'] = {}
-    logger.info('Validating configuration ...')
-    try:
-        validate(conf, CONF_SCHEMA)
-        return conf
-    except ValidationError as exception:
-        logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception)
-        raise ValidationError(
-            best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message
-        )
-
-
-def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any:
-    """
-    Throttles the given callable that it
-    takes at least `min_secs` to finish execution.
-    :param func: Any callable
-    :param min_secs: minimum execution time in seconds
-    :return: Any
-    """
-    start = time.time()
-    result = func(*args, **kwargs)
-    end = time.time()
-    duration = max(min_secs - (end - start), 0.0)
-    logger.debug('Throttling %s for %.2f seconds', func.__name__, duration)
-    time.sleep(duration)
-    return result
-
-
-def common_args_parser(description: str):
-    """
-    Parses given common arguments and returns them as a parsed object.
-    """
-    parser = argparse.ArgumentParser(
-        description=description
-    )
-    parser.add_argument(
-        '-v', '--verbose',
-        help='be verbose',
-        action='store_const',
-        dest='loglevel',
-        const=logging.DEBUG,
-        default=logging.INFO,
-    )
-    parser.add_argument(
-        '--version',
-        action='version',
-        version='%(prog)s {}'.format(__version__),
-    )
-    parser.add_argument(
-        '-c', '--config',
-        help='specify configuration file (default: config.json)',
-        dest='config',
-        default='config.json',
-        type=str,
-        metavar='PATH',
-    )
-    parser.add_argument(
-        '--datadir',
-        help='path to backtest data (default freqdata/tests/testdata)',
-        dest='datadir',
-        default=os.path.join('freqtrade', 'tests', 'testdata'),
-        type=str,
-        metavar='PATH',
-    )
-    parser.add_argument(
-        '-s', '--strategy',
-        help='specify strategy file (default: freqtrade/strategy/default_strategy.py)',
-        dest='strategy',
-        default='default_strategy',
-        type=str,
-        metavar='PATH',
-    )
-    return parser
-
-
-def parse_args(args: List[str], description: str):
-    """
-    Parses given arguments and returns an argparse Namespace instance.
-    Returns None if a sub command has been selected and executed.
-    """
-    parser = common_args_parser(description)
-    parser.add_argument(
-        '--dry-run-db',
-        help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \
-             instead of memory DB. Work only if dry_run is enabled.',
-        action='store_true',
-        dest='dry_run_db',
-    )
-    parser.add_argument(
-        '--dynamic-whitelist',
-        help='dynamically generate and update whitelist \
-             based on 24h BaseVolume (Default 20 currencies)',  # noqa
-        dest='dynamic_whitelist',
-        const=20,
-        type=int,
-        metavar='INT',
-        nargs='?',
-    )
-
-    build_subcommands(parser)
-    return parser.parse_args(args)
-
-
-def scripts_options(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
-        '-p', '--pair',
-        help='Show profits for only this pairs. Pairs are comma-separated.',
-        dest='pair',
-        default=None
-    )
-
-
-def backtesting_options(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
-        '-l', '--live',
-        action='store_true',
-        dest='live',
-        help='using live data',
-    )
-    parser.add_argument(
-        '-i', '--ticker-interval',
-        help='specify ticker interval in minutes (1, 5, 30, 60, 1440)',
-        dest='ticker_interval',
-        type=int,
-        metavar='INT',
-    )
-    parser.add_argument(
-        '--realistic-simulation',
-        help='uses max_open_trades from config to simulate real world limitations',
-        action='store_true',
-        dest='realistic_simulation',
-    )
-    parser.add_argument(
-        '-r', '--refresh-pairs-cached',
-        help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \
-              Use it if you want to run your backtesting with up-to-date data.',
-        action='store_true',
-        dest='refresh_pairs',
-    )
-    parser.add_argument(
-        '--export',
-        help='Export backtest results, argument are: trades\
-              Example --export=trades',
-        type=str,
-        default=None,
-        dest='export',
-    )
-    parser.add_argument(
-        '--timerange',
-        help='Specify what timerange of data to use.',
-        default=None,
-        type=str,
-        dest='timerange',
-    )
-
-
-def hyperopt_options(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
-        '-e', '--epochs',
-        help='specify number of epochs (default: 100)',
-        dest='epochs',
-        default=100,
-        type=int,
-        metavar='INT',
-    )
-    parser.add_argument(
-        '--use-mongodb',
-        help='parallelize evaluations with mongodb (requires mongod in PATH)',
-        dest='mongodb',
-        action='store_true',
-    )
-    parser.add_argument(
-        '-i', '--ticker-interval',
-        help='specify ticker interval in minutes (1, 5, 30, 60, 1440)',
-        dest='ticker_interval',
-        type=int,
-        metavar='INT',
-    )
-    parser.add_argument(
-        '--timerange',
-        help='Specify what timerange of data to use.',
-        default=None,
-        type=str,
-        dest='timerange',
-    )
-
-
-def parse_timerange(text):
-    if text is None:
-        return None
-    syntax = [(r'^-(\d{8})$', (None, 'date')),
-              (r'^(\d{8})-$', ('date', None)),
-              (r'^(\d{8})-(\d{8})$', ('date', 'date')),
-              (r'^(-\d+)$', (None, 'line')),
-              (r'^(\d+)-$', ('line', None)),
-              (r'^(\d+)-(\d+)$', ('index', 'index'))]
-    for rex, stype in syntax:
-        # Apply the regular expression to text
-        match = re.match(rex, text)
-        if match:  # Regex has matched
-            rvals = match.groups()
-            index = 0
-            start = None
-            stop = None
-            if stype[0]:
-                start = rvals[index]
-                if stype[0] != 'date':
-                    start = int(start)
-                index += 1
-            if stype[1]:
-                stop = rvals[index]
-                if stype[1] != 'date':
-                    stop = int(stop)
-            return (stype, start, stop)
-    raise Exception('Incorrect syntax for timerange "%s"' % text)
-
-
-def build_subcommands(parser: argparse.ArgumentParser) -> None:
-    """ Builds and attaches all subcommands """
-    from freqtrade.optimize import backtesting, hyperopt
-
-    subparsers = parser.add_subparsers(dest='subparser')
-
-    # Add backtesting subcommand
-    backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module')
-    backtesting_cmd.set_defaults(func=backtesting.start)
-    backtesting_options(backtesting_cmd)
-
-    # Add hyperopt subcommand
-    hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')
-    hyperopt_cmd.set_defaults(func=hyperopt.start)
-    hyperopt_options(hyperopt_cmd)
-
-
-# Required json-schema for user specified config
-CONF_SCHEMA = {
-    'type': 'object',
-    'properties': {
-        'max_open_trades': {'type': 'integer', 'minimum': 1},
-        'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]},
-        'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']},
-        'stake_amount': {'type': 'number', 'minimum': 0.0005},
-        'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF',
-                                                             'CLP', 'CNY', 'CZK', 'DKK',
-                                                             'EUR', 'GBP', 'HKD', 'HUF',
-                                                             'IDR', 'ILS', 'INR', 'JPY',
-                                                             'KRW', 'MXN', 'MYR', 'NOK',
-                                                             'NZD', 'PHP', 'PKR', 'PLN',
-                                                             'RUB', 'SEK', 'SGD', 'THB',
-                                                             'TRY', 'TWD', 'ZAR', 'USD']},
-        'dry_run': {'type': 'boolean'},
-        'minimal_roi': {
-            'type': 'object',
-            'patternProperties': {
-                '^[0-9.]+$': {'type': 'number'}
-            },
-            'minProperties': 1
-        },
-        'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True},
-        'unfilledtimeout': {'type': 'integer', 'minimum': 0},
-        'bid_strategy': {
-            'type': 'object',
-            'properties': {
-                'ask_last_balance': {
-                    'type': 'number',
-                    'minimum': 0,
-                    'maximum': 1,
-                    'exclusiveMaximum': False
-                },
-            },
-            'required': ['ask_last_balance']
-        },
-        'exchange': {'$ref': '#/definitions/exchange'},
-        'experimental': {
-            'type': 'object',
-            'properties': {
-                'use_sell_signal': {'type': 'boolean'},
-                'sell_profit_only': {'type': 'boolean'}
-            }
-        },
-        'telegram': {
-            'type': 'object',
-            'properties': {
-                'enabled': {'type': 'boolean'},
-                'token': {'type': 'string'},
-                'chat_id': {'type': 'string'},
-            },
-            'required': ['enabled', 'token', 'chat_id']
-        },
-        'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
-        'internals': {
-            'type': 'object',
-            'properties': {
-                'process_throttle_secs': {'type': 'number'},
-                'interval': {'type': 'integer'}
-            }
-        }
-    },
-    'definitions': {
-        'exchange': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'key': {'type': 'string'},
-                'secret': {'type': 'string'},
-                'pair_whitelist': {
-                    'type': 'array',
-                    'items': {
-                        'type': 'string',
-                        'pattern': '^[0-9A-Z]+_[0-9A-Z]+$'
-                    },
-                    'uniqueItems': True
-                },
-                'pair_blacklist': {
-                    'type': 'array',
-                    'items': {
-                        'type': 'string',
-                        'pattern': '^[0-9A-Z]+_[0-9A-Z]+$'
-                    },
-                    'uniqueItems': True
-                }
-            },
-            'required': ['name', 'key', 'secret', 'pair_whitelist']
-        }
-    },
-    'anyOf': [
-        {'required': ['exchange']}
-    ],
-    'required': [
-        'max_open_trades',
-        'stake_currency',
-        'stake_amount',
-        'fiat_display_currency',
-        'dry_run',
-        'bid_strategy',
-        'telegram'
-    ]
-}

freqtrade/tests/test_misc.py

@@ -1,188 +1,31 @@
 # pragma pylint: disable=missing-docstring,C0103
-import argparse
+
-import json
+"""
-import time
+Unit test file for misc.py
-from copy import deepcopy
+"""
-from unittest.mock import MagicMock
 
 import datetime
-import pytest
+from unittest.mock import MagicMock
-from jsonschema import ValidationError
+from freqtrade.analyze import Analyze
-from freqtrade.analyze import parse_ticker_dataframe
+from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, file_dump_json)
-from freqtrade.misc import (common_args_parser, file_dump_json, load_config,
-                            parse_args, parse_timerange, throttle, datesarray_to_datetimearray)
 
 
-def test_throttle():
+def test_shorten_date() -> None:
-
+    """
-    def func():
+    Test shorten_date() function
-        return 42
+    :return: None
-
+    """
-    start = time.time()
+    str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago'
-    result = throttle(func, min_secs=0.1)
+    str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago'
-    end = time.time()
+    assert shorten_date(str_data) == str_shorten_data
-
-    assert result == 42
-    assert end - start > 0.1
-
-    result = throttle(func, min_secs=-1)
-    assert result == 42
-
-
-def test_throttle_with_assets():
-
-    def func(nb_assets=-1):
-        return nb_assets
-
-    result = throttle(func, min_secs=0.1, nb_assets=666)
-    assert result == 666
-
-    result = throttle(func, min_secs=0.1)
-    assert result == -1
-
-
-# Parse common command-line-arguments. Used for all tools
-
-def test_parse_args_none():
-    args = common_args_parser('')
-    assert isinstance(args, argparse.ArgumentParser)
-
-
-def test_parse_args_defaults():
-    args = parse_args([], '')
-    assert args.config == 'config.json'
-    assert args.dynamic_whitelist is None
-    assert args.loglevel == 20
-
-
-def test_parse_args_config():
-    args = parse_args(['-c', '/dev/null'], '')
-    assert args.config == '/dev/null'
-
-    args = parse_args(['--config', '/dev/null'], '')
-    assert args.config == '/dev/null'
-
-
-def test_parse_args_verbose():
-    args = parse_args(['-v'], '')
-    assert args.loglevel == 10
-
-    args = parse_args(['--verbose'], '')
-    assert args.loglevel == 10
-
-
-def test_parse_args_version():
-    with pytest.raises(SystemExit, match=r'0'):
-        parse_args(['--version'], '')
-
-
-def test_parse_args_invalid():
-    with pytest.raises(SystemExit, match=r'2'):
-        parse_args(['-c'], '')
-
-
-# Parse command-line-arguments
-# used for main, backtesting and hyperopt
-
-
-def test_parse_args_dynamic_whitelist():
-    args = parse_args(['--dynamic-whitelist'], '')
-    assert args.dynamic_whitelist == 20
-
-
-def test_parse_args_dynamic_whitelist_10():
-    args = parse_args(['--dynamic-whitelist', '10'], '')
-    assert args.dynamic_whitelist == 10
-
-
-def test_parse_args_dynamic_whitelist_invalid_values():
-    with pytest.raises(SystemExit, match=r'2'):
-        parse_args(['--dynamic-whitelist', 'abc'], '')
-
-
-def test_parse_args_backtesting_invalid():
-    with pytest.raises(SystemExit, match=r'2'):
-        parse_args(['backtesting --ticker-interval'], '')
-
-    with pytest.raises(SystemExit, match=r'2'):
-        parse_args(['backtesting --ticker-interval', 'abc'], '')
-
-
-def test_parse_args_backtesting_custom():
-    args = [
-        '-c', 'test_conf.json',
-        'backtesting',
-        '--live',
-        '--ticker-interval', '1',
-        '--refresh-pairs-cached']
-    call_args = parse_args(args, '')
-    assert call_args.config == 'test_conf.json'
-    assert call_args.live is True
-    assert call_args.loglevel == 20
-    assert call_args.subparser == 'backtesting'
-    assert call_args.func is not None
-    assert call_args.ticker_interval == 1
-    assert call_args.refresh_pairs is True
-
-
-def test_parse_args_hyperopt_custom():
-    args = ['-c', 'test_conf.json', 'hyperopt', '--epochs', '20']
-    call_args = parse_args(args, '')
-    assert call_args.config == 'test_conf.json'
-    assert call_args.epochs == 20
-    assert call_args.loglevel == 20
-    assert call_args.subparser == 'hyperopt'
-    assert call_args.func is not None
-
-
-def test_file_dump_json(mocker):
-    file_open = mocker.patch('freqtrade.misc.open', MagicMock())
-    json_dump = mocker.patch('json.dump', MagicMock())
-    file_dump_json('somefile', [1, 2, 3])
-    assert file_open.call_count == 1
-    assert json_dump.call_count == 1
-
-
-def test_parse_timerange_incorrect():
-    assert ((None, 'line'), None, -200) == parse_timerange('-200')
-    assert (('line', None), 200, None) == parse_timerange('200-')
-    with pytest.raises(Exception, match=r'Incorrect syntax.*'):
-        parse_timerange('-')
-
-
-def test_load_config(default_conf, mocker):
-    file_mock = mocker.patch('freqtrade.misc.open', mocker.mock_open(
-        read_data=json.dumps(default_conf)
-    ))
-    validated_conf = load_config('somefile')
-    assert file_mock.call_count == 1
-    assert validated_conf.items() >= default_conf.items()
-
-
-def test_load_config_invalid_pair(default_conf, mocker):
-    conf = deepcopy(default_conf)
-    conf['exchange']['pair_whitelist'].append('BTC-ETH')
-    mocker.patch(
-        'freqtrade.misc.open',
-        mocker.mock_open(
-            read_data=json.dumps(conf)))
-    with pytest.raises(ValidationError, match=r'.*does not match.*'):
-        load_config('somefile')
-
-
-def test_load_config_missing_attributes(default_conf, mocker):
-    conf = deepcopy(default_conf)
-    conf.pop('exchange')
-    mocker.patch(
-        'freqtrade.misc.open',
-        mocker.mock_open(
-            read_data=json.dumps(conf)))
-    with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'):
-        load_config('somefile')
 
 
 def test_datesarray_to_datetimearray(ticker_history):
-    dataframes = parse_ticker_dataframe(ticker_history)
+    """
+    Test datesarray_to_datetimearray() function
+    :return: None
+    """
+    dataframes = Analyze.parse_ticker_dataframe(ticker_history)
     dates = datesarray_to_datetimearray(dataframes['date'])
 
     assert isinstance(dates[0], datetime.datetime)
@@ -194,3 +37,15 @@ def test_datesarray_to_datetimearray(ticker_history):
 
     date_len = len(dates)
     assert date_len == 3
+
+
+def test_file_dump_json(mocker):
+    """
+    Test file_dump_json()
+    :return: None
+    """
+    file_open = mocker.patch('freqtrade.misc.open', MagicMock())
+    json_dump = mocker.patch('json.dump', MagicMock())
+    file_dump_json('somefile', [1, 2, 3])
+    assert file_open.call_count == 1
+    assert json_dump.call_count == 1