diff --git a/.gitignore b/.gitignore index 672dd1f6d..c81b55222 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,5 @@ target/ .venv .idea .vscode + +hyperopt_trials.pickle diff --git a/.pylintrc b/.pylintrc index 65ed64830..dce99c067 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,10 +1,10 @@ [MASTER] -extension-pkg-whitelist=numpy,talib +extension-pkg-whitelist=numpy,talib,talib.abstract [BASIC] good-names=logger ignore=vendor [TYPECHECK] -ignored-modules=numpy,talib +ignored-modules=numpy,talib,talib.abstract diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 4b16550c5..f85c46248 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -4,14 +4,14 @@ Functions to analyze ticker data with indicators and produce buy and sell signal import logging from datetime import timedelta from enum import Enum -from typing import List, Dict +from typing import Dict, List import arrow import talib.abstract as ta from pandas import DataFrame, to_datetime -from freqtrade.exchange import get_ticker_history import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.exchange import get_ticker_history logger = logging.getLogger(__name__) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 4883db037..a5fd68016 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -1,7 +1,9 @@ import logging -from typing import List, Dict, Optional +import requests +from typing import Dict, List, Optional -from bittrex.bittrex import Bittrex as _Bittrex, API_V2_0, API_V1_1 +from bittrex.bittrex import Bittrex as _Bittrex +from bittrex.bittrex import API_V1_1, API_V2_0 from requests.exceptions import ContentDecodingError from freqtrade import OperationalException @@ -13,6 +15,20 @@ _API: _Bittrex = None _API_V2: _Bittrex = None _EXCHANGE_CONF: dict = {} +# API socket timeout +API_TIMEOUT = 60 + + +def custom_requests(request_url, apisign): + """ + Set timeout for requests + """ + return requests.get( + request_url, + headers={"apisign": apisign}, + timeout=API_TIMEOUT + ).json() + class Bittrex(Exchange): """ @@ -31,12 +47,14 @@ class Bittrex(Exchange): api_secret=_EXCHANGE_CONF['secret'], calls_per_second=1, api_version=API_V1_1, + dispatch=custom_requests ) _API_V2 = _Bittrex( api_key=_EXCHANGE_CONF['key'], api_secret=_EXCHANGE_CONF['secret'], calls_per_second=1, api_version=API_V2_0, + dispatch=custom_requests ) self.cached_ticker = {} diff --git a/freqtrade/exchange/interface.py b/freqtrade/exchange/interface.py index 1be84abe5..6121a98b3 100644 --- a/freqtrade/exchange/interface.py +++ b/freqtrade/exchange/interface.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Dict, Optional +from typing import Dict, List, Optional class Exchange(ABC): diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 3f0b6330b..0132e531d 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -1,5 +1,6 @@ import logging import time + from pymarketcap import Pymarketcap logger = logging.getLogger(__name__) diff --git a/freqtrade/main.py b/freqtrade/main.py index efd0d7c44..c404d6c11 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -5,43 +5,25 @@ import logging import sys import time import traceback -import arrow from datetime import datetime -from typing import Dict, Optional, List +from typing import Dict, List, Optional +import arrow import requests -from requests.adapters import TimeoutSauce from cachetools import cached, TTLCache -from freqtrade import __version__, exchange, persistence, rpc, DependencyException, \ - OperationalException -from freqtrade.analyze import get_signal, SignalType -from freqtrade.misc import State, get_state, update_state, parse_args, throttle, \ - load_config -from freqtrade.persistence import Trade +from freqtrade import (DependencyException, OperationalException, __version__, + exchange, persistence, rpc) +from freqtrade.analyze import SignalType, get_signal from freqtrade.fiat_convert import CryptoToFiatConverter +from freqtrade.misc import (State, get_state, load_config, parse_args, + throttle, update_state) +from freqtrade.persistence import Trade logger = logging.getLogger('freqtrade') _CONF = {} -DEFAULT_TIMEOUT = 120 - - -# Set requests default timeout (fix for #127) -class DefaultTimeout(TimeoutSauce): - def __init__(self, *args, **kwargs): - connect = kwargs.get('connect', DEFAULT_TIMEOUT) - read = kwargs.get('read', connect) - if connect is None: - connect = DEFAULT_TIMEOUT - if read is None: - read = connect - super(DefaultTimeout, self).__init__(connect=connect, read=read) - - -requests.adapters.TimeoutSauce = DefaultTimeout - def refresh_whitelist(whitelist: List[str]) -> List[str]: """ @@ -197,11 +179,11 @@ def execute_sell(trade: Trade, limit: float) -> None: profit_trade = trade.calc_profit(rate=limit) message = '*{exchange}:* Selling [{pair}]({pair_url}) with limit `{limit:.8f}`'.format( - exchange=trade.exchange, - pair=trade.pair.replace('_', '/'), - pair_url=exchange.get_pair_detail_url(trade.pair), - limit=limit - ) + exchange=trade.exchange, + pair=trade.pair.replace('_', '/'), + pair_url=exchange.get_pair_detail_url(trade.pair), + limit=limit + ) # For regular case, when the configuration exists if 'stake_currency' in _CONF and 'fiat_display_currency' in _CONF: @@ -213,12 +195,12 @@ def execute_sell(trade: Trade, limit: float) -> None: ) message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ '` / {profit_fiat:.3f} {fiat})`'.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade, - coin=_CONF['stake_currency'], - profit_fiat=profit_fiat, - fiat=_CONF['fiat_display_currency'], + gain="profit" if fmt_exp_profit > 0 else "loss", + profit_percent=fmt_exp_profit, + profit_coin=profit_trade, + coin=_CONF['stake_currency'], + profit_fiat=profit_fiat, + fiat=_CONF['fiat_display_currency'], ) # Because telegram._forcesell does not have the configuration # Ignore the FIAT value and does not show the stake_currency as well diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 9c581bd47..5f094933a 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -3,10 +3,11 @@ import enum import json import logging import time -from typing import Any, Callable, List, Dict +import os +from typing import Any, Callable, Dict, List -from jsonschema import validate, Draft4Validator -from jsonschema.exceptions import best_match, ValidationError +from jsonschema import Draft4Validator, validate +from jsonschema.exceptions import ValidationError, best_match from wrapt import synchronized from freqtrade import __version__ @@ -62,8 +63,8 @@ def load_config(path: str) -> Dict: try: validate(conf, CONF_SCHEMA) return conf - except ValidationError as ex: - logger.fatal('Invalid configuration. See config.json.example. Reason: %s', ex) + except ValidationError as exception: + logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception) raise ValidationError( best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message ) @@ -86,7 +87,7 @@ def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: return result -def parse_args_common(args: List[str], description: str): +def common_args_parser(description: str): """ Parses given common arguments and returns them as a parsed object. """ @@ -122,11 +123,11 @@ def parse_args(args: List[str], description: str): Parses given arguments and returns an argparse Namespace instance. Returns None if a sub command has been selected and executed. """ - parser = parse_args_common(args, description) + parser = common_args_parser(description) parser.add_argument( '--dry-run-db', - help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" instead of memory DB. Work only if dry_run is \ - enabled.', # noqa + help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \ + instead of memory DB. Work only if dry_run is enabled.', action='store_true', dest='dry_run_db', ) @@ -134,13 +135,14 @@ def parse_args(args: List[str], description: str): '-dd', '--datadir', help='path to backtest data (default freqdata/tests/testdata', dest='datadir', - default='freqtrade/tests/testdata', + default=os.path.join('freqtrade', 'tests', 'testdata'), type=str, metavar='PATH', ) parser.add_argument( '--dynamic-whitelist', - help='dynamically generate and update whitelist based on 24h BaseVolume (Default 20 currencies)', # noqa + help='dynamically generate and update whitelist \ + based on 24h BaseVolume (Default 20 currencies)', # noqa dest='dynamic_whitelist', const=20, type=int, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index dd813cd33..ab12fe559 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1,19 +1,19 @@ # pragma pylint: disable=missing-docstring,W0212 import logging -from typing import Tuple, Dict +from typing import Dict, Tuple import arrow from pandas import DataFrame, Series from tabulate import tabulate +import freqtrade.misc as misc +import freqtrade.optimize as optimize from freqtrade import exchange from freqtrade.analyze import populate_buy_trend, populate_sell_trend from freqtrade.exchange import Bittrex from freqtrade.main import min_roi_reached -import freqtrade.misc as misc from freqtrade.optimize import preprocess -import freqtrade.optimize as optimize from freqtrade.persistence import Trade logger = logging.getLogger(__name__) @@ -93,15 +93,14 @@ def get_trade_entry(pair, row, ticker, trade_count_lock, args): if min_roi_reached(trade, row2.close, row2.date) or \ (row2.sell == 1 and use_sell_signal) or \ current_profit_percent <= stoploss: - current_profit_btc = trade.calc_profit(rate=row2.close) - - return row2.Index, (pair, - current_profit_percent, - current_profit_btc, - row2.Index - row.Index, - current_profit_btc > 0, - current_profit_btc < 0 - ) + current_profit_btc = trade.calc_profit(rate=row2.close) + return row2.Index, (pair, + current_profit_percent, + current_profit_btc, + row2.Index - row.Index, + current_profit_btc > 0, + current_profit_btc < 0 + ) def backtest(args) -> DataFrame: @@ -217,6 +216,6 @@ def start(args): 'record': args.export }) logger.info( - '\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa + '\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa generate_text_table(data, results, config['stake_currency'], args.ticker_interval) ) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 146874b9c..07060ef85 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,14 +4,18 @@ import json import logging import sys +import pickle +import signal +import os from functools import reduce from math import exp from operator import itemgetter -from hyperopt import fmin, tpe, hp, Trials, STATUS_OK, STATUS_FAIL, space_eval +from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from hyperopt.mongoexp import MongoTrials from pandas import DataFrame +from freqtrade import main # noqa from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.misc import load_config @@ -27,7 +31,7 @@ logger = logging.getLogger(__name__) # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data TARGET_TRADES = 1100 -TOTAL_TRIES = None +TOTAL_TRIES = 0 _CURRENT_TRIES = 0 CURRENT_BEST_LOSS = 100 @@ -43,6 +47,10 @@ EXPECTED_MAX_PROFIT = 3.85 PROCESSED = None # optimize.preprocess(optimize.load_data()) OPTIMIZE_CONFIG = hyperopt_optimize_conf() +# Hyperopt Trials +TRIALS_FILE = os.path.join('freqtrade', 'optimize', 'hyperopt_trials.pickle') +TRIALS = Trials() + # Monkey patch config from freqtrade import main # noqa main._CONF = OPTIMIZE_CONFIG @@ -99,6 +107,26 @@ SPACE = { } +def save_trials(trials, trials_path=TRIALS_FILE): + """Save hyperopt trials to file""" + logger.info('Saving Trials to \'{}\''.format(trials_path)) + pickle.dump(trials, open(trials_path, 'wb')) + + +def read_trials(trials_path=TRIALS_FILE): + """Read hyperopt trials file""" + logger.info('Reading Trials from \'{}\''.format(trials_path)) + trials = pickle.load(open(trials_path, 'rb')) + os.remove(trials_path) + return trials + + +def log_trials_result(trials): + vals = json.dumps(trials.best_trial['misc']['vals'], indent=4) + results = trials.best_trial['result']['result'] + logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -169,7 +197,7 @@ def format_results(results: DataFrame): results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), results.duration.mean() * 5, - ) + ) def buy_strategy_generator(params): @@ -218,7 +246,8 @@ def buy_strategy_generator(params): def start(args): - global TOTAL_TRIES, PROCESSED, SPACE + global TOTAL_TRIES, PROCESSED, SPACE, TRIALS, _CURRENT_TRIES + TOTAL_TRIES = args.epochs exchange._API = Bittrex({'key': '', 'secret': ''}) @@ -240,9 +269,19 @@ def start(args): logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!') db_name = 'freqtrade_hyperopt' - trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') + TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') else: - trials = Trials() + logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, signal_handler) + # read trials file if we have one + if os.path.exists(TRIALS_FILE): + TRIALS = read_trials() + + _CURRENT_TRIES = len(TRIALS.results) + TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES + logger.info( + 'Continuing with trials. Current: {}, Total: {}' + .format(_CURRENT_TRIES, TOTAL_TRIES)) try: best_parameters = fmin( @@ -250,10 +289,10 @@ def start(args): space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, - trials=trials + trials=TRIALS ) - results = sorted(trials.results, key=itemgetter('loss')) + results = sorted(TRIALS.results, key=itemgetter('loss')) best_result = results[0]['result'] except ValueError: @@ -267,3 +306,15 @@ def start(args): logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) logger.info('Best Result:\n%s', best_result) + + # Store trials result to file to resume next time + save_trials(TRIALS) + + +def signal_handler(sig, frame): + """Hyperopt SIGINT handler""" + logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) + + save_trials(TRIALS) + log_trials_result(TRIALS) + sys.exit(0) diff --git a/freqtrade/optimize/hyperopt_conf.py b/freqtrade/optimize/hyperopt_conf.py index cbd95973a..8e044e549 100644 --- a/freqtrade/optimize/hyperopt_conf.py +++ b/freqtrade/optimize/hyperopt_conf.py @@ -15,10 +15,10 @@ def hyperopt_optimize_conf() -> dict: 'stake_currency': 'BTC', 'stake_amount': 0.01, "minimal_roi": { - '40': 0.0, - '30': 0.01, - '20': 0.02, - '0': 0.04, + '40': 0.0, + '30': 0.01, + '20': 0.02, + '0': 0.04, }, 'stoploss': -0.10, "bid_strategy": { diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index d50c9acb4..0cac3bbe9 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -1,10 +1,11 @@ import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Optional, Dict +from typing import Dict, Optional import arrow -from sqlalchemy import Boolean, Column, DateTime, Float, Integer, String, create_engine +from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, + create_engine) from sqlalchemy.engine import Engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4d7957f59..009714682 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1,21 +1,21 @@ import logging import re +from datetime import datetime, timedelta from decimal import Decimal -from datetime import timedelta, datetime -from typing import Callable, Any +from typing import Any, Callable import arrow from pandas import DataFrame from sqlalchemy import and_, func, text from tabulate import tabulate -from telegram import ParseMode, Bot, Update, ReplyKeyboardMarkup +from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater -from freqtrade import exchange, __version__ -from freqtrade.misc import get_state, State, update_state -from freqtrade.persistence import Trade +from freqtrade import __version__, exchange from freqtrade.fiat_convert import CryptoToFiatConverter +from freqtrade.misc import State, get_state, update_state +from freqtrade.persistence import Trade # Remove noisy log messages logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) @@ -255,7 +255,7 @@ def _daily(bot: Bot, update: Update) -> None: ), symbol=_CONF['fiat_display_currency'] ) - ] + ] for key, value in profit_days.items() ] stats = tabulate(stats, diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 788585345..c779aa726 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -2,10 +2,10 @@ from datetime import datetime from unittest.mock import MagicMock -import pytest import arrow +import pytest from jsonschema import validate -from telegram import Message, Chat, Update +from telegram import Chat, Message, Update from freqtrade.misc import CONF_SCHEMA @@ -20,10 +20,10 @@ def default_conf(): "fiat_display_currency": "USD", "dry_run": True, "minimal_roi": { - "40": 0.0, - "30": 0.01, - "20": 0.02, - "0": 0.04 + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 }, "stoploss": -0.10, "unfilledtimeout": 600, diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index 704ca0f6e..e01e7fa02 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -32,7 +32,7 @@ def _stub_config(): 'secret': ''} -class Fake_bittrex(): +class FakeBittrex(): def __init__(self, success=True): self.success = True # Believe in yourself self.result = None @@ -145,7 +145,7 @@ def test_exchange_bittrex_fee(): def test_exchange_bittrex_buy_good(mocker): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() uuid = wb.buy('BTC_ETH', 1, 1) assert uuid == fb.fake_buysell_limit(1, 2, 3)['result']['uuid'] @@ -156,7 +156,7 @@ def test_exchange_bittrex_buy_good(mocker): def test_exchange_bittrex_sell_good(mocker): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() uuid = wb.sell('BTC_ETH', 1, 1) assert uuid == fb.fake_buysell_limit(1, 2, 3)['result']['uuid'] @@ -167,7 +167,7 @@ def test_exchange_bittrex_sell_good(mocker): def test_exchange_bittrex_get_balance(mocker): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() bal = wb.get_balance('BTC_ETH') assert bal == fb.fake_get_balance(1)['result']['Balance'] @@ -178,7 +178,7 @@ def test_exchange_bittrex_get_balance(mocker): def test_exchange_bittrex_get_balances(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() bals = wb.get_balances() assert bals == fb.fake_get_balances()['result'] @@ -189,7 +189,7 @@ def test_exchange_bittrex_get_balances(): def test_exchange_bittrex_get_ticker(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() # Poll ticker, which updates the cache tick = wb.get_ticker('BTC_ETH') @@ -210,7 +210,7 @@ def test_exchange_bittrex_get_ticker(): def test_exchange_bittrex_get_ticker_bad(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() fb.result = {'success': True, 'result': {'Bid': 1}} # incomplete result with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'): @@ -222,15 +222,15 @@ def test_exchange_bittrex_get_ticker_bad(): wb.get_ticker('BTC_ETH') -def test_exchange_bittrex_get_ticker_historyOne(): +def test_exchange_bittrex_get_ticker_history_one(): wb = make_wrap_bittrex() - Fake_bittrex() + FakeBittrex() assert wb.get_ticker_history('BTC_ETH', 1) def test_exchange_bittrex_get_ticker_history(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() assert wb.get_ticker_history('BTC_ETH', 5) with pytest.raises(ValueError, match=r'.*Cannot parse tick_interval.*'): wb.get_ticker_history('BTC_ETH', 2) @@ -253,7 +253,7 @@ def test_exchange_bittrex_get_ticker_history(): def test_exchange_bittrex_get_order(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() order = wb.get_order('someUUID') assert order['id'] == 'ABC123' fb.success = False @@ -263,7 +263,7 @@ def test_exchange_bittrex_get_order(): def test_exchange_bittrex_cancel_order(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() wb.cancel_order('someUUID') with pytest.raises(btx.OperationalException, match=r'no such order'): fb.success = False @@ -284,7 +284,7 @@ def test_exchange_get_pair_detail_url(): def test_exchange_get_markets(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() x = wb.get_markets() assert x == ['__'] with pytest.raises(btx.OperationalException, match=r'market gone'): @@ -294,7 +294,7 @@ def test_exchange_get_markets(): def test_exchange_get_market_summaries(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() assert ['sum'] == wb.get_market_summaries() with pytest.raises(btx.OperationalException, match=r'no summary'): fb.success = False @@ -303,7 +303,7 @@ def test_exchange_get_market_summaries(): def test_exchange_get_wallet_health(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() x = wb.get_wallet_health() assert x[0]['Currency'] == 'BTC_ETH' with pytest.raises(btx.OperationalException, match=r'bad health'): diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index a309af7fe..3e03d26c0 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,7 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 - from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ - log_results + log_results, save_trials, read_trials def test_loss_calculation_prefer_correct_trade_count(): @@ -27,16 +26,37 @@ def test_loss_calculation_has_limited_profit(): def create_trials(mocker): + """ + When creating trials, mock the hyperopt Trials so that *by default* + - we don't create any pickle'd files in the filesystem + - we might have a pickle'd file so make sure that we return + false when looking for it + """ + mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', + return_value='freqtrade/tests/optimize/ut_trials.pickle') + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', + return_value=False) + mocker.patch('freqtrade.optimize.hyperopt.save_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.read_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.os.remove', + return_value=True) return mocker.Mock( results=[{ 'loss': 1, - 'result': 'foo' - }] + 'result': 'foo', + 'status': 'ok' + }], + best_trial={'misc': {'vals': {'adx': 999}}} ) def test_start_calls_fmin(mocker): - mocker.patch('freqtrade.optimize.hyperopt.Trials', return_value=create_trials(mocker)) + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials) + mocker.patch('freqtrade.optimize.hyperopt.sorted', + return_value=trials.results) mocker.patch('freqtrade.optimize.preprocess') mocker.patch('freqtrade.optimize.load_data') mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) @@ -141,3 +161,63 @@ def test_fmin_throw_value_error(mocker, caplog): for line in exists: assert line in caplog.text + + +def test_resuming_previous_hyperopt_results_succeeds(mocker): + import freqtrade.optimize.hyperopt as hyperopt + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.TRIALS', + return_value=trials) + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', + return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.len', + return_value=len(trials.results)) + mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials', + return_value=trials) + mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.sorted', + return_value=trials.results) + mocker.patch('freqtrade.optimize.preprocess') + mocker.patch('freqtrade.optimize.load_data') + mocker.patch('freqtrade.optimize.hyperopt.fmin', + return_value={}) + args = mocker.Mock(epochs=1, + config='config.json.example', + mongodb=False) + + start(args) + + mock_read.assert_called_once() + mock_save.assert_called_once() + + current_tries = hyperopt._CURRENT_TRIES + total_tries = hyperopt.TOTAL_TRIES + + assert current_tries == len(trials.results) + assert total_tries == (current_tries + len(trials.results)) + + +def test_save_trials_saves_trials(mocker): + trials = create_trials(mocker) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', + return_value=None) + trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', + return_value='ut_trials.pickle') + mocker.patch('freqtrade.optimize.hyperopt.open', + return_value=trials_path) + save_trials(trials, trials_path) + + mock_dump.assert_called_once_with(trials, trials_path) + + +def test_read_trials_returns_trials_file(mocker): + trials = create_trials(mocker) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', + return_value=trials) + mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', + return_value=mock_load) + + assert read_trials() == trials + mock_open.assert_called_once() + mock_load.assert_called_once() diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index a5892f278..57c41c9c6 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -6,7 +6,7 @@ from shutil import copyfile from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ - download_backtesting_testdata, load_tickerdata_file + download_backtesting_testdata, load_tickerdata_file # Change this if modifying BTC_UNITEST testdatafile _btc_unittest_length = 13681 diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index be0bd47e8..7c42c676e 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -7,17 +7,17 @@ from freqtrade.main import refresh_whitelist, gen_pair_whitelist def whitelist_conf(): return { - "stake_currency": "BTC", - "exchange": { - "pair_whitelist": [ - "BTC_ETH", - "BTC_TKN", - "BTC_TRST", - "BTC_SWT", - "BTC_BCC" + 'stake_currency': 'BTC', + 'exchange': { + 'pair_whitelist': [ + 'BTC_ETH', + 'BTC_TKN', + 'BTC_TRST', + 'BTC_SWT', + 'BTC_BCC' ], - "pair_blacklist": [ - "BTC_BLK" + 'pair_blacklist': [ + 'BTC_BLK' ], }, } @@ -25,52 +25,51 @@ def whitelist_conf(): def get_market_summaries(): return [{ - "MarketName": "BTC-TKN", - "High": 0.00000919, - "Low": 0.00000820, - "Volume": 74339.61396015, - "Last": 0.00000820, - "BaseVolume": 1664, - "TimeStamp": "2014-07-09T07:19:30.15", - "Bid": 0.00000820, - "Ask": 0.00000831, - "OpenBuyOrders": 15, - "OpenSellOrders": 15, - "PrevDay": 0.00000821, - "Created": "2014-03-20T06:00:00", - "DisplayMarketName": "" - }, { - "MarketName": "BTC-ETH", - "High": 0.00000072, - "Low": 0.00000001, - "Volume": 166340678.42280999, - "Last": 0.00000005, - "BaseVolume": 42, - "TimeStamp": "2014-07-09T07:21:40.51", - "Bid": 0.00000004, - "Ask": 0.00000005, - "OpenBuyOrders": 18, - "OpenSellOrders": 18, - "PrevDay": 0.00000002, - "Created": "2014-05-30T07:57:49.637", - "DisplayMarketName": "" - }, { - "MarketName": "BTC-BLK", - "High": 0.00000072, - "Low": 0.00000001, - "Volume": 166340678.42280999, - "Last": 0.00000005, - "BaseVolume": 3, - "TimeStamp": "2014-07-09T07:21:40.51", - "Bid": 0.00000004, - "Ask": 0.00000005, - "OpenBuyOrders": 18, - "OpenSellOrders": 18, - "PrevDay": 0.00000002, - "Created": "2014-05-30T07:57:49.637", - "DisplayMarketName": "" - } - ] + 'MarketName': 'BTC-TKN', + 'High': 0.00000919, + 'Low': 0.00000820, + 'Volume': 74339.61396015, + 'Last': 0.00000820, + 'BaseVolume': 1664, + 'TimeStamp': '2014-07-09T07:19:30.15', + 'Bid': 0.00000820, + 'Ask': 0.00000831, + 'OpenBuyOrders': 15, + 'OpenSellOrders': 15, + 'PrevDay': 0.00000821, + 'Created': '2014-03-20T06:00:00', + 'DisplayMarketName': '' + }, { + 'MarketName': 'BTC-ETH', + 'High': 0.00000072, + 'Low': 0.00000001, + 'Volume': 166340678.42280999, + 'Last': 0.00000005, + 'BaseVolume': 42, + 'TimeStamp': '2014-07-09T07:21:40.51', + 'Bid': 0.00000004, + 'Ask': 0.00000005, + 'OpenBuyOrders': 18, + 'OpenSellOrders': 18, + 'PrevDay': 0.00000002, + 'Created': '2014-05-30T07:57:49.637', + 'DisplayMarketName': '' + }, { + 'MarketName': 'BTC-BLK', + 'High': 0.00000072, + 'Low': 0.00000001, + 'Volume': 166340678.42280999, + 'Last': 0.00000005, + 'BaseVolume': 3, + 'TimeStamp': '2014-07-09T07:21:40.51', + 'Bid': 0.00000004, + 'Ask': 0.00000005, + 'OpenBuyOrders': 18, + 'OpenSellOrders': 18, + 'PrevDay': 0.00000002, + 'Created': '2014-05-30T07:57:49.637', + 'DisplayMarketName': '' + }] def get_health(): @@ -95,7 +94,8 @@ def test_refresh_market_pair_not_in_whitelist(mocker): mocker.patch.dict('freqtrade.main._CONF', conf) mocker.patch.multiple('freqtrade.main.exchange', get_wallet_health=get_health) - refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist'] + ['BTC_XXX']) + refreshedwhitelist = refresh_whitelist( + conf['exchange']['pair_whitelist'] + ['BTC_XXX']) # List ordered by BaseVolume whitelist = ['BTC_ETH', 'BTC_TKN'] # Ensure all except those in whitelist are removed @@ -123,7 +123,8 @@ def test_refresh_whitelist_dynamic(mocker): get_market_summaries=get_market_summaries) # argument: use the whitelist dynamically by exchange-volume whitelist = ['BTC_TKN', 'BTC_ETH'] - refreshedwhitelist = refresh_whitelist(gen_pair_whitelist(conf['stake_currency'])) + refreshedwhitelist = refresh_whitelist( + gen_pair_whitelist(conf['stake_currency'])) assert whitelist == refreshedwhitelist diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 040c45f26..d1afc4200 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -6,8 +6,9 @@ import arrow import pytest from pandas import DataFrame -from freqtrade.analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators, \ - get_signal, SignalType, populate_sell_trend +from freqtrade.analyze import (SignalType, get_signal, parse_ticker_dataframe, + populate_buy_trend, populate_indicators, + populate_sell_trend) @pytest.fixture diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index cd8af2c52..f9230a03f 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -1,7 +1,7 @@ import pandas -from freqtrade import analyze import freqtrade.optimize +from freqtrade import analyze _pairs = ['BTC_ETH'] diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 4aa9b1c4f..ddc1c8e29 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -1,10 +1,11 @@ # pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 import time -import pytest from unittest.mock import MagicMock -from freqtrade.fiat_convert import CryptoToFiatConverter, CryptoFiat +import pytest + +from freqtrade.fiat_convert import CryptoFiat, CryptoToFiatConverter def test_pair_convertion_object(): diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index cceb555f7..97bef2257 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -1,28 +1,27 @@ # pragma pylint: disable=missing-docstring,C0103 import copy +import logging from unittest.mock import MagicMock +import arrow import pytest import requests -import logging -import arrow from sqlalchemy import create_engine +import freqtrade.main as main from freqtrade import DependencyException, OperationalException from freqtrade.analyze import SignalType from freqtrade.exchange import Exchanges -from freqtrade.main import create_trade, handle_trade, init, \ - get_target_bid, _process, execute_sell, check_handle_timedout -from freqtrade.misc import get_state, State +from freqtrade.main import (_process, check_handle_timedout, create_trade, + execute_sell, get_target_bid, handle_trade, init) +from freqtrade.misc import State, get_state from freqtrade.persistence import Trade -import freqtrade.main as main -# Test that main() can start backtesting or hyperopt. -# and also ensure we can pass some specific arguments -# argument parsing is done in test_misc.py - def test_parse_args_backtesting(mocker): + """ Test that main() can start backtesting or hyperopt. + and also ensure we can pass some specific arguments + argument parsing is done in test_misc.py """ backtesting_mock = mocker.patch( 'freqtrade.optimize.backtesting.start', MagicMock()) with pytest.raises(SystemExit, match=r'0'): @@ -269,7 +268,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): assert trade.close_date is not None -def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog): +def test_handle_trade_roi(default_conf, ticker, mocker, caplog): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -301,7 +300,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog) assert ('freqtrade', logging.DEBUG, 'Executing sell due to ROI ...') in caplog.record_tuples -def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker, caplog): +def test_handle_trade_experimental(default_conf, ticker, mocker, caplog): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -353,7 +352,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mo handle_trade(trade) -def test_check_handle_timedout_buy(default_conf, ticker, health, limit_buy_order_old, mocker): +def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) cancel_order_mock = MagicMock() mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) @@ -385,7 +384,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, health, limit_buy_order assert len(trades) == 0 -def test_check_handle_timedout_sell(default_conf, ticker, health, limit_sell_order_old, mocker): +def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) cancel_order_mock = MagicMock() mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) @@ -418,7 +417,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, health, limit_sell_ord def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, - health, mocker): + mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) cancel_order_mock = MagicMock() mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) @@ -624,54 +623,54 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker): def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': True, - } + default_conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': True, + } - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.min_roi_reached', return_value=False) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value='mocked_limit_buy')) - init(default_conf, create_engine('sqlite://')) - create_trade(0.001) + init(default_conf, create_engine('sqlite://')) + create_trade(0.001) - trade = Trade.query.first() - trade.update(limit_buy_order) - assert handle_trade(trade) is False + trade = Trade.query.first() + trade.update(limit_buy_order) + assert handle_trade(trade) is False def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': False, - } + default_conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': False, + } - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.min_roi_reached', return_value=False) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value='mocked_limit_buy')) - init(default_conf, create_engine('sqlite://')) - create_trade(0.001) + init(default_conf, create_engine('sqlite://')) + create_trade(0.001) - trade = Trade.query.first() - trade.update(limit_buy_order) - assert handle_trade(trade) is True + trade = Trade.query.first() + trade.update(limit_buy_order) + assert handle_trade(trade) is True diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index e0cd70709..6b93fb8cb 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -1,15 +1,15 @@ # pragma pylint: disable=missing-docstring,C0103 +import argparse import json import time -import argparse from copy import deepcopy import pytest from unittest.mock import MagicMock from jsonschema import ValidationError -from freqtrade.misc import throttle, parse_args, load_config,\ - parse_args_common, file_dump_json +from freqtrade.misc import (common_args_parser, load_config, parse_args, + throttle, file_dump_json) def test_throttle(): @@ -40,12 +40,10 @@ def test_throttle_with_assets(): assert result == -1 -# Parse common command-line-arguments -# used for all tools - +# Parse common command-line-arguments. Used for all tools def test_parse_args_none(): - args = parse_args_common([], '') + args = common_args_parser('') assert isinstance(args, argparse.ArgumentParser) @@ -88,12 +86,12 @@ def test_parse_args_invalid(): def test_parse_args_dynamic_whitelist(): args = parse_args(['--dynamic-whitelist'], '') - assert args.dynamic_whitelist is 20 + assert args.dynamic_whitelist == 20 def test_parse_args_dynamic_whitelist_10(): args = parse_args(['--dynamic-whitelist', '10'], '') - assert args.dynamic_whitelist is 10 + assert args.dynamic_whitelist == 10 def test_parse_args_dynamic_whitelist_invalid_values(): diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index c2e2c13ea..70797f960 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,9 +1,10 @@ # pragma pylint: disable=missing-docstring +import os + import pytest -import os from freqtrade.exchange import Exchanges -from freqtrade.persistence import init, Trade +from freqtrade.persistence import Trade, init def test_init_create_session(default_conf, mocker): diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 44961891c..ce636a4b5 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -6,11 +6,11 @@ import matplotlib # Install PYQT5 manually if you want to test this helper func matplotlib.use("Qt5Agg") import matplotlib.pyplot as plt from freqtrade import exchange, analyze -from freqtrade.misc import parse_args_common +from freqtrade.misc import common_args_parser def plot_parse_args(args ): - parser = parse_args_common(args, 'Graph utility') + parser = common_args_parser(args, 'Graph utility') parser.add_argument( '-p', '--pair', help = 'What currency pair',