diff --git a/.gitignore b/.gitignore index c1bb0be68..e5ac932f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # Freqtrade rules freqtrade/tests/testdata/*.json hyperopt_conf.py -config.json +config*.json *.sqlite .hyperopt logfile.txt diff --git a/Dockerfile b/Dockerfile index 918552526..afafd93c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6.4-slim-stretch +FROM python:3.6.5-slim-stretch # Install TA-lib RUN apt-get update && apt-get -y install curl build-essential && apt-get clean diff --git a/config_full.json.example b/config_full.json.example index 5362ee943..77ef0faa0 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -48,5 +48,7 @@ "initial_state": "running", "internals": { "process_throttle_secs": 5 - } + }, + "strategy": "DefaultStrategy", + "strategy_path": "/some/folder/" } diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 6911e9e20..b9ff3fe40 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -14,12 +14,12 @@ Since the version `0.16.0` the bot allows using custom strategy file. This is very simple. Copy paste your strategy file into the folder `user_data/strategies`. -Let assume you have a strategy file `awesome-strategy.py`: +Let assume you have a class called `AwesomeStrategy` in the file `awesome-strategy.py`: 1. Move your file into `user_data/strategies` (you should have `user_data/strategies/awesome-strategy.py` -2. Start the bot with the param `--strategy awesome-strategy` (the parameter is the name of the file without '.py') +2. Start the bot with the param `--strategy AwesomeStrategy` (the parameter is the class name) ```bash -python3 ./freqtrade/main.py --strategy awesome_strategy +python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` ## Change your strategy @@ -35,11 +35,18 @@ A strategy file contains all the information needed to build a good strategy: - Stoploss recommended - Hyperopt parameter -The bot also include a sample strategy you can update: `user_data/strategies/test_strategy.py`. -You can test it with the parameter: `--strategy test_strategy` +The bot also include a sample strategy called `TestStrategy` you can update: `user_data/strategies/test_strategy.py`. +You can test it with the parameter: `--strategy TestStrategy` ```bash -python3 ./freqtrade/main.py --strategy awesome_strategy +python3 ./freqtrade/main.py --strategy AwesomeStrategy +``` + +### Specify custom strategy location +If you want to use a strategy from a different folder you can pass `--strategy-path` + +```bash +python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/folder ``` **For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index ce5bb46a9..075413b21 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -26,9 +26,9 @@ optional arguments: --version show program's version number and exit -c PATH, --config PATH specify configuration file (default: config.json) - -s PATH, --strategy PATH - specify strategy file (default: - freqtrade/strategy/default_strategy.py) + -s NAME, --strategy NAME + specify strategy class name (default: DefaultStrategy) + --strategy-path PATH specify additional strategy lookup path --dry-run-db Force dry run to use a local DB "tradesv3.dry_run.sqlite" instead of memory DB. Work only if dry_run is enabled. @@ -48,21 +48,19 @@ python3 ./freqtrade/main.py -c path/far/far/away/config.json ``` ### How to use --strategy? -This parameter will allow you to load your custom strategy file. Per -default without `--strategy` or `-s` the bot will load the -`default_strategy` included with the bot (`freqtrade/strategy/default_strategy.py`). +This parameter will allow you to load your custom strategy class. +Per default without `--strategy` or `-s` the bot will load the +`DefaultStrategy` included with the bot (`freqtrade/strategy/default_strategy.py`). -The bot will search your strategy file into `user_data/strategies` and -`freqtrade/strategy`. +The bot will search your strategy file within `user_data/strategies` and `freqtrade/strategy`. -To load a strategy, simply pass the file name (without .py) in this -parameters. +To load a strategy, simply pass the class name (e.g.: `CustomStrategy`) in this parameter. **Example:** -In `user_data/strategies` you have a file `my_awesome_strategy.py` to -load it: +In `user_data/strategies` you have a file `my_awesome_strategy.py` which has +a strategy class called `AwesomeStrategy` to load it: ```bash -python3 ./freqtrade/main.py --strategy my_awesome_strategy +python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` If the bot does not find your strategy file, it will display in an error @@ -70,9 +68,16 @@ message the reason (File not found, or errors in your code). Learn more about strategy file in [optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md). +### How to use --strategy-path? +This parameter allows you to add an additional strategy lookup path, which gets +checked before the default locations (The passed path must be a folder!): +```bash +python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/folder +``` + #### How to install a strategy? This is very simple. Copy paste your strategy file into the folder -`user_data/strategies`. And voila, the bot is ready to use it. +`user_data/strategies` or use `--strategy-path`. And voila, the bot is ready to use it. ### How to use --dynamic-whitelist? Per default `--dynamic-whitelist` will retrieve the 20 currencies based diff --git a/docs/configuration.md b/docs/configuration.md index 2beab348d..3d36947c3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -35,6 +35,8 @@ The table below will list all configuration parameters. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. | `initial_state` | running | No | Defines the initial application state. More information below. +| `strategy` | DefaultStrategy | No | Defines Strategy class to use. +| `strategy_path` | null | No | Adds an additional strategy lookup path (must be a folder). | `internals.process_throttle_secs` | 5 | Yes | Set the process throttle. Value in second. The definition of each config parameters is in diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 9e83f004e..e041fa2ff 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -127,3 +127,14 @@ Day Profit BTC Profit USD ## /version > **Version:** `0.14.3` + +### using proxy with telegram +in [freqtrade/freqtrade/rpc/telegram.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/rpc/telegram.py) replace +``` +self._updater = Updater(token=self._config['telegram']['token'], workers=0) +``` + +with +``` +self._updater = Updater(token=self._config['telegram']['token'], request_kwargs={'proxy_url': 'socks5://127.0.0.1:1080/'}, workers=0) +``` diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 16f8b3b4a..c630d06b2 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -16,9 +16,9 @@ class OperationalException(BaseException): """ -class NetworkException(BaseException): +class TemporaryError(BaseException): """ - Network related error. + Temporary network or exchange related error. This could happen when an exchange is congested, unavailable, or the user has networking problems. Usually resolves itself after a time. """ diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b843db3f4..dcb5376ce 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -9,10 +9,10 @@ from typing import Dict, List, Tuple import arrow from pandas import DataFrame, to_datetime +from freqtrade import constants from freqtrade.exchange import get_ticker_history from freqtrade.persistence import Trade -from freqtrade.strategy.strategy import Strategy -from freqtrade.constants import Constants +from freqtrade.strategy.resolver import StrategyResolver logger = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class Analyze(object): :param config: Bot configuration (use the one from Configuration()) """ self.config = config - self.strategy = Strategy(self.config) + self.strategy = StrategyResolver(self.config).strategy @staticmethod def parse_ticker_dataframe(ticker: list) -> DataFrame: @@ -54,7 +54,14 @@ class Analyze(object): utc=True, infer_datetime_format=True) - frame.sort_values('date', inplace=True) + # group by index and aggregate results to eliminate duplicate ticks + frame = frame.groupby(by='date', as_index=False, sort=True).agg({ + 'open': 'first', + 'high': 'max', + 'low': 'min', + 'close': 'last', + 'volume': 'max', + }) return frame def populate_indicators(self, dataframe: DataFrame) -> DataFrame: @@ -139,7 +146,7 @@ class Analyze(object): # Check if dataframe is out of date signal_date = arrow.get(latest['date']) - interval_minutes = Constants.TICKER_INTERVAL_MINUTES[interval] + interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] if signal_date < arrow.utcnow() - timedelta(minutes=(interval_minutes + 5)): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index e82ec05b5..357bcb937 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -9,8 +9,7 @@ import re import arrow from typing import List, Tuple, Optional -from freqtrade import __version__ -from freqtrade.constants import Constants +from freqtrade import __version__, constants class Arguments(object): @@ -81,9 +80,16 @@ class Arguments(object): ) self.parser.add_argument( '-s', '--strategy', - help='specify strategy file (default: %(default)s)', + help='specify strategy class name (default: %(default)s)', dest='strategy', - default='default_strategy', + default='DefaultStrategy', + type=str, + metavar='NAME', + ) + self.parser.add_argument( + '--strategy-path', + help='specify additional strategy lookup path', + dest='strategy_path', type=str, metavar='PATH', ) @@ -92,7 +98,7 @@ class Arguments(object): help='dynamically generate and update whitelist \ based on 24h BaseVolume (Default 20 currencies)', # noqa dest='dynamic_whitelist', - const=Constants.DYNAMIC_WHITELIST, + const=constants.DYNAMIC_WHITELIST, type=int, metavar='INT', nargs='?', @@ -163,7 +169,7 @@ class Arguments(object): '-e', '--epochs', help='specify number of epochs (default: %(default)d)', dest='epochs', - default=Constants.HYPEROPT_EPOCH, + default=constants.HYPEROPT_EPOCH, type=int, metavar='INT', ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 32ebabe18..43d0a0bf9 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -10,8 +10,7 @@ from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match import ccxt -from freqtrade import OperationalException -from freqtrade.constants import Constants +from freqtrade import OperationalException, constants logger = logging.getLogger(__name__) @@ -34,8 +33,12 @@ class Configuration(object): logger.info('Using config: %s ...', self.args.config) config = self._load_config_file(self.args.config) - # Add the strategy file to use - config.update({'strategy': self.args.strategy}) + # Set strategy if not specified in config and or if it's non default + if self.args.strategy != constants.DEFAULT_STRATEGY or not config.get('strategy'): + config.update({'strategy': self.args.strategy}) + + if self.args.strategy_path: + config.update({'strategy_path': self.args.strategy_path}) # Load Common configuration config = self._load_common_config(config) @@ -186,7 +189,7 @@ class Configuration(object): :return: Returns the config if valid, otherwise throw an exception """ try: - validate(conf, Constants.CONF_SCHEMA) + validate(conf, constants.CONF_SCHEMA) return conf except ValidationError as exception: logger.fatal( @@ -194,7 +197,7 @@ class Configuration(object): exception ) raise ValidationError( - best_match(Draft4Validator(Constants.CONF_SCHEMA).iter_errors(conf)).message + best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message ) def get_config(self) -> Dict[str, Any]: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 802785891..fa09d8eab 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -1,136 +1,131 @@ # pragma pylint: disable=too-few-public-methods """ -List bot constants +bot constants """ +DYNAMIC_WHITELIST = 20 # pairs +PROCESS_THROTTLE_SECS = 5 # sec +TICKER_INTERVAL = 5 # min +HYPEROPT_EPOCH = 100 # epochs +RETRY_TIMEOUT = 30 # sec +DEFAULT_STRATEGY = 'DefaultStrategy' + +TICKER_INTERVAL_MINUTES = { + '1m': 1, + '5m': 5, + '15m': 15, + '30m': 30, + '1h': 60, + '2h': 120, + '4h': 240, + '6h': 360, + '12h': 720, + '1d': 1440, + '1w': 10080, +} -class Constants(object): - """ - Static class that contain all bot constants - """ - DYNAMIC_WHITELIST = 20 # pairs - PROCESS_THROTTLE_SECS = 5 # sec - TICKER_INTERVAL = 5 # min - HYPEROPT_EPOCH = 100 # epochs - RETRY_TIMEOUT = 30 # sec - DEFAULT_STRATEGY = 'default_strategy' - - TICKER_INTERVAL_MINUTES = { - '1m': 1, - '5m': 5, - '15m': 15, - '30m': 30, - '1h': 60, - '2h': 120, - '4h': 240, - '6h': 360, - '12h': 720, - '1d': 1440, - '1w': 10080, - } - - # Required json-schema for user specified config - CONF_SCHEMA = { - 'type': 'object', - 'properties': { - 'max_open_trades': {'type': 'integer', 'minimum': 1}, - 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, - 'stake_amount': {'type': 'number', 'minimum': 0.0005}, - 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', - 'CLP', 'CNY', 'CZK', 'DKK', - 'EUR', 'GBP', 'HKD', 'HUF', - 'IDR', 'ILS', 'INR', 'JPY', - 'KRW', 'MXN', 'MYR', 'NOK', - 'NZD', 'PHP', 'PKR', 'PLN', - 'RUB', 'SEK', 'SGD', 'THB', - 'TRY', 'TWD', 'ZAR', 'USD']}, - 'dry_run': {'type': 'boolean'}, - 'minimal_roi': { - 'type': 'object', - 'patternProperties': { - '^[0-9.]+$': {'type': 'number'} +# Required json-schema for user specified config +CONF_SCHEMA = { + 'type': 'object', + 'properties': { + 'max_open_trades': {'type': 'integer', 'minimum': 1}, + 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, + 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, + 'stake_amount': {'type': 'number', 'minimum': 0.0005}, + 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', + 'CLP', 'CNY', 'CZK', 'DKK', + 'EUR', 'GBP', 'HKD', 'HUF', + 'IDR', 'ILS', 'INR', 'JPY', + 'KRW', 'MXN', 'MYR', 'NOK', + 'NZD', 'PHP', 'PKR', 'PLN', + 'RUB', 'SEK', 'SGD', 'THB', + 'TRY', 'TWD', 'ZAR', 'USD']}, + 'dry_run': {'type': 'boolean'}, + 'minimal_roi': { + 'type': 'object', + 'patternProperties': { + '^[0-9.]+$': {'type': 'number'} + }, + 'minProperties': 1 + }, + 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, + 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, + 'bid_strategy': { + 'type': 'object', + 'properties': { + 'ask_last_balance': { + 'type': 'number', + 'minimum': 0, + 'maximum': 1, + 'exclusiveMaximum': False }, - 'minProperties': 1 }, - 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, - 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, - 'bid_strategy': { - 'type': 'object', - 'properties': { - 'ask_last_balance': { - 'type': 'number', - 'minimum': 0, - 'maximum': 1, - 'exclusiveMaximum': False - }, - }, - 'required': ['ask_last_balance'] - }, - 'exchange': {'$ref': '#/definitions/exchange'}, - 'experimental': { - 'type': 'object', - 'properties': { - 'use_sell_signal': {'type': 'boolean'}, - 'sell_profit_only': {'type': 'boolean'} - } - }, - 'telegram': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'token': {'type': 'string'}, - 'chat_id': {'type': 'string'}, - }, - 'required': ['enabled', 'token', 'chat_id'] - }, - 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, - 'internals': { - 'type': 'object', - 'properties': { - 'process_throttle_secs': {'type': 'number'}, - 'interval': {'type': 'integer'} - } + 'required': ['ask_last_balance'] + }, + 'exchange': {'$ref': '#/definitions/exchange'}, + 'experimental': { + 'type': 'object', + 'properties': { + 'use_sell_signal': {'type': 'boolean'}, + 'sell_profit_only': {'type': 'boolean'} } }, - 'definitions': { - 'exchange': { - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'key': {'type': 'string'}, - 'secret': {'type': 'string'}, - 'pair_whitelist': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' - }, - 'uniqueItems': True - }, - 'pair_blacklist': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' - }, - 'uniqueItems': True - } - }, - 'required': ['name', 'key', 'secret', 'pair_whitelist'] - } + 'telegram': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'token': {'type': 'string'}, + 'chat_id': {'type': 'string'}, + }, + 'required': ['enabled', 'token', 'chat_id'] }, - 'anyOf': [ - {'required': ['exchange']} - ], - 'required': [ - 'max_open_trades', - 'stake_currency', - 'stake_amount', - 'fiat_display_currency', - 'dry_run', - 'bid_strategy', - 'telegram' - ] - } + 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, + 'internals': { + 'type': 'object', + 'properties': { + 'process_throttle_secs': {'type': 'number'}, + 'interval': {'type': 'integer'} + } + } + }, + 'definitions': { + 'exchange': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'key': {'type': 'string'}, + 'secret': {'type': 'string'}, + 'pair_whitelist': { + 'type': 'array', + 'items': { + 'type': 'string', + 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' + }, + 'uniqueItems': True + }, + 'pair_blacklist': { + 'type': 'array', + 'items': { + 'type': 'string', + 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' + }, + 'uniqueItems': True + } + }, + 'required': ['name', 'key', 'secret', 'pair_whitelist'] + } + }, + 'anyOf': [ + {'required': ['exchange']} + ], + 'required': [ + 'max_open_trades', + 'stake_currency', + 'stake_amount', + 'fiat_display_currency', + 'dry_run', + 'bid_strategy', + 'telegram' + ] +} diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 2601cb836..d0d6d4c04 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -3,11 +3,12 @@ import logging from random import randint from typing import List, Dict, Any, Optional +from datetime import datetime import ccxt import arrow -from freqtrade import OperationalException, DependencyException, NetworkException +from freqtrade import OperationalException, DependencyException, TemporaryError from freqtrade.constants import Constants logger = logging.getLogger(__name__) @@ -15,7 +16,7 @@ logger = logging.getLogger(__name__) # Current selected exchange _API: ccxt.Exchange = None -_CONF: dict = {} +_CONF: Dict = {} API_RETRY_COUNT = 4 # Holds all open sell orders for dry_run @@ -33,15 +34,16 @@ def retrier(f): count = kwargs.pop('count', API_RETRY_COUNT) try: return f(*args, **kwargs) - except (NetworkException, DependencyException) as ex: - logger.warning('%s returned exception: "%s"', f, ex) + except (TemporaryError, DependencyException) as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) if count > 0: count -= 1 kwargs.update({'count': count}) - logger.warning('retrying %s still for %s times', f, count) + logger.warning('retrying %s() still for %s times', f.__name__, count) return wrapper(*args, **kwargs) else: - raise OperationalException('Giving up retrying: %s', f) + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex return wrapper @@ -144,7 +146,8 @@ def buy(pair: str, rate: float, amount: float) -> Dict: 'side': 'buy', 'remaining': 0.0, 'datetime': arrow.utcnow().isoformat(), - 'status': 'closed' + 'status': 'closed', + 'fee': None } return {'id': order_id} @@ -162,10 +165,10 @@ def buy(pair: str, rate: float, amount: float) -> Dict: 'Tried to buy amount {} at rate {} (total {}).' 'Message: {}'.format(pair, amount, rate, rate*amount, e) ) - except ccxt.NetworkError as e: - raise NetworkException( - 'Could not place buy order due to networking error. Message: {}'.format(e) - ) + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not place buy order due to {}. Message: {}'.format( + e.__class__.__name__, e)) except ccxt.BaseError as e: raise OperationalException(e) @@ -200,23 +203,30 @@ def sell(pair: str, rate: float, amount: float) -> Dict: 'Tried to sell amount {} at rate {} (total {}).' 'Message: {}'.format(pair, amount, rate, rate*amount, e) ) - except ccxt.NetworkError as e: - raise NetworkException( - 'Could not place sell order due to networking error. Message: {}'.format(e) - ) + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not place sell order due to {}. Message: {}'.format( + e.__class__.__name__, e)) except ccxt.BaseError as e: raise OperationalException(e) +@retrier def get_balance(currency: str) -> float: if _CONF['dry_run']: return 999.9 # ccxt exception is already handled by get_balances balances = get_balances() - return balances[currency]['free'] + balance = balances.get(currency) + if balance is None: + raise TemporaryError( + 'Could not get {} balance due to malformed exchange response: {}'.format( + currency, balances)) + return balance['free'] +@retrier def get_balances() -> dict: if _CONF['dry_run']: return {} @@ -230,10 +240,10 @@ def get_balances() -> dict: balances.pop("used", None) return balances - except ccxt.NetworkError as e: - raise NetworkException( - 'Could not get balance due to networking error. Message: {}'.format(e) - ) + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not get balance due to {}. Message: {}'.format( + e.__class__.__name__, e)) except ccxt.BaseError as e: raise OperationalException(e) @@ -242,17 +252,17 @@ def get_balances() -> dict: def get_tickers() -> Dict: try: return _API.fetch_tickers() - except ccxt.NetworkError as e: - raise NetworkException( - 'Could not load tickers due to networking error. Message: {}'.format(e) - ) - except ccxt.BaseError as e: - raise OperationalException(e) except ccxt.NotSupported as e: raise OperationalException( 'Exchange {} does not support fetching tickers in batch.' 'Message: {}'.format(_API.name, e) ) + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not load tickers due to {}. Message: {}'.format( + e.__class__.__name__, e)) + except ccxt.BaseError as e: + raise OperationalException(e) # TODO: remove refresh argument, keeping it to keep track of where it was intended to be used @@ -260,10 +270,10 @@ def get_tickers() -> Dict: def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict: try: return _API.fetch_ticker(pair) - except ccxt.NetworkError as e: - raise NetworkException( - 'Could not load tickers due to networking error. Message: {}'.format(e) - ) + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not load ticker history due to {}. Message: {}'.format( + e.__class__.__name__, e)) except ccxt.BaseError as e: raise OperationalException(e) @@ -296,38 +306,39 @@ def get_ticker_history(pair: str, tick_interval: str, since_ms: Optional[int] = since_ms = data[-1][0] + 1 return data - - except ccxt.NetworkError as e: - raise NetworkException( - 'Could not load ticker history due to networking error. Message: {}'.format(e) - ) - except ccxt.BaseError as e: - raise OperationalException('Could not fetch ticker data. Msg: {}'.format(e)) except ccxt.NotSupported as e: raise OperationalException( 'Exchange {} does not support fetching historical candlestick data.' 'Message: {}'.format(_API.name, e) ) + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not load ticker history due to {}. Message: {}'.format( + e.__class__.__name__, e)) + except ccxt.BaseError as e: + raise OperationalException('Could not fetch ticker data. Msg: {}'.format(e)) +@retrier def cancel_order(order_id: str, pair: str) -> None: if _CONF['dry_run']: return try: return _API.cancel_order(order_id, pair) - except ccxt.NetworkError as e: - raise NetworkException( - 'Could not get order due to networking error. Message: {}'.format(e) - ) except ccxt.InvalidOrder as e: raise DependencyException( 'Could not cancel order. Message: {}'.format(e) ) + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not cancel order due to {}. Message: {}'.format( + e.__class__.__name__, e)) except ccxt.BaseError as e: raise OperationalException(e) +@retrier def get_order(order_id: str, pair: str) -> Dict: if _CONF['dry_run']: order = _DRY_RUN_OPEN_ORDERS[order_id] @@ -337,14 +348,34 @@ def get_order(order_id: str, pair: str) -> Dict: return order try: return _API.fetch_order(order_id, pair) - except ccxt.NetworkError as e: - raise NetworkException( - 'Could not get order due to networking error. Message: {}'.format(e) - ) except ccxt.InvalidOrder as e: raise DependencyException( 'Could not get order. Message: {}'.format(e) ) + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not get order due to {}. Message: {}'.format( + e.__class__.__name__, e)) + except ccxt.BaseError as e: + raise OperationalException(e) + + +@retrier +def get_trades_for_order(order_id: str, pair: str, since: datetime) -> List: + if _CONF['dry_run']: + return [] + if not exchange_has('fetchMyTrades'): + return [] + try: + my_trades = _API.fetch_my_trades(pair, since.timestamp()) + matched_trades = [trade for trade in my_trades if trade['order'] == order_id] + + return matched_trades + + except ccxt.NetworkError as e: + raise TemporaryError( + 'Could not get trades due to networking error. Message: {}'.format(e) + ) except ccxt.BaseError as e: raise OperationalException(e) @@ -360,13 +391,14 @@ def get_pair_detail_url(pair: str) -> str: return "" +@retrier def get_markets() -> List[dict]: try: return _API.fetch_markets() - except ccxt.NetworkError as e: - raise NetworkException( - 'Could not load markets due to networking error. Message: {}'.format(e) - ) + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not load markets due to {}. Message: {}'.format( + e.__class__.__name__, e)) except ccxt.BaseError as e: raise OperationalException(e) @@ -379,11 +411,29 @@ def get_id() -> str: return _API.id +@retrier def get_fee(symbol='ETH/BTC', type='', side='', amount=1, price=1, taker_or_maker='maker') -> float: - # validate that markets are loaded before trying to get fee - if _API.markets is None or len(_API.markets) == 0: - _API.load_markets() + try: + # validate that markets are loaded before trying to get fee + if _API.markets is None or len(_API.markets) == 0: + _API.load_markets() - return _API.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, - price=price, takerOrMaker=taker_or_maker)['rate'] + return _API.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, + price=price, takerOrMaker=taker_or_maker)['rate'] + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not get fee info due to {}. Message: {}'.format( + e.__class__.__name__, e)) + except ccxt.BaseError as e: + raise OperationalException(e) + + +def get_amount_lots(pair: str, amount: float) -> float: + """ + get buyable amount rounding, .. + """ + # validate that markets are loaded before trying to get fee + if not _API.markets: + _API.load_markets() + return _API.amount_to_lots(pair, amount) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 947d20df3..0a332b952 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -3,7 +3,6 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() """ import copy -import json import logging import time import traceback @@ -12,19 +11,19 @@ from typing import Dict, List, Optional, Any, Callable import arrow import requests -from cachetools import cached, TTLCache +from cachetools import TTLCache, cached from freqtrade import ( - DependencyException, OperationalException, exchange, persistence, __version__ + DependencyException, OperationalException, TemporaryError, + exchange, persistence, __version__, ) +from freqtrade import constants from freqtrade.analyze import Analyze -from freqtrade.constants import Constants from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.persistence import Trade from freqtrade.rpc.rpc_manager import RPCManager from freqtrade.state import State - logger = logging.getLogger(__name__) @@ -111,7 +110,7 @@ class FreqtradeBot(object): elif state == State.RUNNING: min_secs = self.config.get('internals', {}).get( 'process_throttle_secs', - Constants.PROCESS_THROTTLE_SECS + constants.PROCESS_THROTTLE_SECS ) nb_assets = self.config.get('dynamic_whitelist', None) @@ -173,9 +172,9 @@ class FreqtradeBot(object): self.check_handle_timedout(self.config['unfilledtimeout']) Trade.session.flush() - except (requests.exceptions.RequestException, json.JSONDecodeError) as error: + except TemporaryError as error: logger.warning('%s, retrying in 30 seconds...', error) - time.sleep(Constants.RETRY_TIMEOUT) + time.sleep(constants.RETRY_TIMEOUT) except OperationalException: self.rpc.send_msg( '*Status:* OperationalException:\n```\n{traceback}```{hint}' @@ -287,7 +286,7 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - # Pick pair based on StochRSI buy signals + # Pick pair based on buy signals for _pair in whitelist: (buy, sell) = self.analyze.get_signal(_pair, interval) if buy and not sell: @@ -323,11 +322,13 @@ class FreqtradeBot(object): ) ) # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL + fee = exchange.get_fee(symbol=pair, taker_or_maker='maker') trade = Trade( pair=pair, stake_amount=stake_amount, amount=amount, - fee=exchange.get_fee(taker_or_maker='maker'), + fee_open=fee, + fee_close=fee, open_rate=buy_limit, open_date=datetime.utcnow(), exchange=exchange.get_id(), @@ -358,17 +359,74 @@ class FreqtradeBot(object): Tries to execute a sell trade :return: True if executed """ - # Get order details for actual price per unit - if trade.open_order_id: - # Update trade with order values - logger.info('Found open order for %s', trade) - trade.update(exchange.get_order(trade.open_order_id, trade.pair)) + try: + # Get order details for actual price per unit + if trade.open_order_id: + # Update trade with order values + logger.info('Found open order for %s', trade) + order = exchange.get_order(trade.open_order_id, trade.pair) + # Try update amount (binance-fix) + try: + new_amount = self.get_real_amount(trade, order) + if order['amount'] != new_amount: + order['amount'] = new_amount + # Fee was applied, so set to 0 + trade.fee_open = 0 - if trade.is_open and trade.open_order_id is None: - # Check if we can sell our current pair - return self.handle_trade(trade) + except OperationalException as exception: + logger.warning("could not update trade amount: %s", exception) + + trade.update(order) + + if trade.is_open and trade.open_order_id is None: + # Check if we can sell our current pair + return self.handle_trade(trade) + except DependencyException as exception: + logger.warning('Unable to sell trade: %s', exception) return False + def get_real_amount(self, trade: Trade, order: Dict) -> float: + """ + Get real amount for the trade + Necessary for exchanges which charge fees in base currency (e.g. binance) + """ + order_amount = order['amount'] + # Only run for closed orders + if trade.fee_open == 0 or order['status'] == 'open': + return order_amount + + # use fee from order-dict if possible + if 'fee' in order and order['fee']: + if trade.pair.startswith(order['fee']['currency']): + new_amount = order_amount - order['fee']['cost'] + logger.info("Applying fee on amount for %s (from %s to %s) from Order", + trade, order['amount'], new_amount) + return new_amount + + # Fallback to Trades + trades = exchange.get_trades_for_order(trade.open_order_id, trade.pair, trade.open_date) + + if len(trades) == 0: + logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade) + return order_amount + amount = 0 + fee_abs = 0 + for exectrade in trades: + amount += exectrade['amount'] + if "fee" in exectrade: + # only applies if fee is in quote currency! + if trade.pair.startswith(exectrade['fee']['currency']): + fee_abs += exectrade['fee']['cost'] + + if amount != order_amount: + logger.warning("amount {} does not match amount {}".format(amount, trade.amount)) + raise OperationalException("Half bought? Amounts don't match") + real_amount = amount - fee_abs + if fee_abs != 0: + logger.info("Applying fee on amount for {} (from {} to {}) from Trades".format( + trade, order['amount'], real_amount)) + return real_amount + def handle_trade(self, trade: Trade) -> bool: """ Sells the current pair if the threshold is reached and updates the trade record. @@ -388,7 +446,7 @@ class FreqtradeBot(object): if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): self.execute_sell(trade, current_rate) return True - + logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False def check_handle_timedout(self, timeoutvalue: int) -> None: @@ -426,7 +484,7 @@ class FreqtradeBot(object): """Buy timeout - cancel order :return: True if order was fully cancelled """ - exchange.cancel_order(trade.open_order_id) + exchange.cancel_order(trade.open_order_id, trade.pair) if order['remaining'] == order['amount']: # if trade is not partially completed, just delete the trade Trade.session.delete(trade) @@ -456,7 +514,7 @@ class FreqtradeBot(object): """ if order['remaining'] == order['amount']: # if trade is not partially completed, just cancel the trade - exchange.cancel_order(trade.open_order_id) + exchange.cancel_order(trade.open_order_id, trade.pair) trade.close_rate = None trade.close_profit = None trade.close_date = None diff --git a/freqtrade/main.py b/freqtrade/main.py index 93d7e5c54..9639922f9 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -3,7 +3,6 @@ Main Freqtrade bot script. Read the documentation to know what cli arguments you need. """ - import logging import sys from typing import List @@ -30,9 +29,10 @@ def main(sysargv: List[str]) -> None: # Means if Backtesting or Hyperopt have been called we exit the bot if hasattr(args, 'func'): args.func(args) - return 0 + return freqtrade = None + return_code = 1 try: # Load and validate configuration config = Configuration(args).get_config() @@ -46,12 +46,13 @@ def main(sysargv: List[str]) -> None: except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') + return_code = 0 except BaseException: logger.exception('Fatal exception!') finally: if freqtrade: freqtrade.clean() - sys.exit(0) + sys.exit(return_code) def set_loggers() -> None: diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index fcbe0dc5b..a93564fd4 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -183,11 +183,11 @@ def load_cached_data_for_updating(filename: str, return (data, since_ms) -# FIX: 20180110, suggest rename interval to tick_interval def download_backtesting_testdata(datadir: str, pair: str, tick_interval: str = '5m', - timerange: Optional[Tuple[Tuple, int, int]] = None) -> bool: + timerange: Optional[Tuple[Tuple, int, int]] = None) -> None: + """ Download the latest ticker intervals from the exchange for the pairs passed in parameters The data is downloaded starting from the last correct ticker interval data that @@ -198,7 +198,8 @@ def download_backtesting_testdata(datadir: str, :param pairs: list of pairs to download :param tick_interval: ticker interval :param timerange: range of time to download - :return: bool + :return: None + """ path = make_testdata_path(datadir) @@ -223,5 +224,3 @@ def download_backtesting_testdata(datadir: str, logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) misc.file_dump_json(filename, data) - - return True diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1bd091155..376730d0f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -4,11 +4,12 @@ This module contains the backtesting logic """ import logging +import operator from argparse import Namespace from typing import Dict, Tuple, Any, List, Optional import arrow -from pandas import DataFrame, Series +from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize @@ -19,7 +20,6 @@ from freqtrade.configuration import Configuration from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade - logger = logging.getLogger(__name__) @@ -66,11 +66,12 @@ class Backtesting(object): :param data: dictionary with preprocessed backtesting data :return: tuple containing min_date, max_date """ - all_dates = Series([]) - for pair_data in data.values(): - all_dates = all_dates.append(pair_data['date']) - all_dates.sort_values(inplace=True) - return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) + timeframe = [ + (arrow.get(min(frame.date)), arrow.get(max(frame.date))) + for frame in data.values() + ] + return min(timeframe, key=operator.itemgetter(0))[0], \ + max(timeframe, key=operator.itemgetter(1))[1] def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: """ @@ -113,12 +114,14 @@ class Backtesting(object): stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) + fee = exchange.get_fee() trade = Trade( open_rate=buy_row.close, open_date=buy_row.date, stake_amount=stake_amount, amount=stake_amount / buy_row.open, - fee=exchange.get_fee() + fee_open=fee, + fee_close=fee ) # calculate win/lose forwards from buy point @@ -199,9 +202,9 @@ class Backtesting(object): # record a tuple of pair, current_profit_percent, # entry-date, duration records.append((pair, trade_entry[1], - row.date.timestamp(), - row2.date.timestamp(), - row.date, trade_entry[3])) + row.date.strftime('%s'), + row2.date.strftime('%s'), + index, trade_entry[3])) # For now export inside backtest(), maybe change so that backtest() # returns a tuple like: (dataframe, records, logs, etc) if record and record.find('trades') >= 0: @@ -302,12 +305,9 @@ def start(args: Namespace) -> None: :param args: Cli args from Arguments() :return: None """ - - # Initialize logger - logger.info('Starting freqtrade in Backtesting mode') - # Initialize configuration config = setup_configuration(args) + logger.info('Starting freqtrade in Backtesting mode') # Initialize backtesting object backtesting = Backtesting(config) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3bdd47eb6..20fa5380d 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -29,7 +29,6 @@ from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting from user_data.hyperopt_conf import hyperopt_optimize_conf - logger = logging.getLogger(__name__) @@ -591,11 +590,11 @@ def start(args: Namespace) -> None: logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) - logger.info('Starting freqtrade in Hyperopt mode') - # Initialize configuration # Monkey patch the configuration with hyperopt_conf.py configuration = Configuration(args) + logger.info('Starting freqtrade in Hyperopt mode') + optimize_config = hyperopt_optimize_conf() config = configuration._load_common_config(optimize_config) config = configuration._load_backtesting_config(config) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 55f53329a..ed81ad2ec 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -85,7 +85,8 @@ class Trade(_DECL_BASE): exchange = Column(String, nullable=False) pair = Column(String, nullable=False) is_open = Column(Boolean, nullable=False, default=True) - fee = Column(Float, nullable=False, default=0.0) + fee_open = Column(Float, nullable=False, default=0.0) + fee_close = Column(Float, nullable=False, default=0.0) open_rate = Column(Float) close_rate = Column(Float) close_profit = Column(Float) @@ -156,7 +157,7 @@ class Trade(_DECL_BASE): getcontext().prec = 8 buy_trade = (Decimal(self.amount) * Decimal(self.open_rate)) - fees = buy_trade * Decimal(fee or self.fee) + fees = buy_trade * Decimal(fee or self.fee_open) return float(buy_trade + fees) def calc_close_trade_price( @@ -177,7 +178,7 @@ class Trade(_DECL_BASE): return 0.0 sell_trade = (Decimal(self.amount) * Decimal(rate or self.close_rate)) - fees = sell_trade * Decimal(fee or self.fee) + fees = sell_trade * Decimal(fee or self.fee_close) return float(sell_trade - fees) def calc_profit( @@ -195,7 +196,7 @@ class Trade(_DECL_BASE): open_trade_price = self.calc_open_trade_price() close_trade_price = self.calc_close_trade_price( rate=(rate or self.close_rate), - fee=(fee or self.fee) + fee=(fee or self.fee_close) ) return float("{0:.8f}".format(close_trade_price - open_trade_price)) @@ -215,7 +216,7 @@ class Trade(_DECL_BASE): open_trade_price = self.calc_open_trade_price() close_trade_price = self.calc_close_trade_price( rate=(rate or self.close_rate), - fee=(fee or self.fee) + fee=(fee or self.fee_close) ) return float("{0:.8f}".format((close_trade_price / open_trade_price) - 1)) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index ec39578c7..22689f17a 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -7,8 +7,6 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.indicator_helpers import fishers_inverse from freqtrade.strategy.interface import IStrategy -class_name = 'DefaultStrategy' - class DefaultStrategy(IStrategy): """ diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4eb73fb2e..dcf665a02 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -33,7 +33,6 @@ class IStrategy(ABC): Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame :return: DataFrame with buy column - :return: """ @abstractmethod @@ -41,5 +40,5 @@ class IStrategy(ABC): """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame - :return: DataFrame with buy column + :return: DataFrame with sell column """ diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py new file mode 100644 index 000000000..8f4972919 --- /dev/null +++ b/freqtrade/strategy/resolver.py @@ -0,0 +1,130 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom strategies +""" +import importlib.util +import inspect +import logging +import os +from collections import OrderedDict +from typing import Optional, Dict, Type + +from freqtrade import constants +from freqtrade.strategy.interface import IStrategy + + +logger = logging.getLogger(__name__) + + +class StrategyResolver(object): + """ + This class contains all the logic to load custom strategy class + """ + + __slots__ = ['strategy'] + + def __init__(self, config: Optional[Dict] = None) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + config = config or {} + + # Verify the strategy is in the configuration, otherwise fallback to the default strategy + strategy_name = config.get('strategy') or constants.DEFAULT_STRATEGY + self.strategy = self._load_strategy(strategy_name, extra_dir=config.get('strategy_path')) + + # Set attributes + # Check if we need to override configuration + if 'minimal_roi' in config: + self.strategy.minimal_roi = config['minimal_roi'] + logger.info("Override strategy \'minimal_roi\' with value in config file.") + + if 'stoploss' in config: + self.strategy.stoploss = config['stoploss'] + logger.info( + "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] + ) + + if 'ticker_interval' in config: + self.strategy.ticker_interval = config['ticker_interval'] + logger.info( + "Override strategy \'ticker_interval\' with value in config file: %s.", + config['ticker_interval'] + ) + + # Sort and apply type conversions + self.strategy.minimal_roi = OrderedDict(sorted( + {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), + key=lambda t: t[0])) + self.strategy.stoploss = float(self.strategy.stoploss) + + def _load_strategy( + self, strategy_name: str, extra_dir: Optional[str] = None) -> Optional[IStrategy]: + """ + Search and loads the specified strategy. + :param strategy_name: name of the module to import + :param extra_dir: additional directory to search for the given strategy + :return: Strategy instance or None + """ + current_path = os.path.dirname(os.path.realpath(__file__)) + abs_paths = [ + os.path.join(current_path, '..', '..', 'user_data', 'strategies'), + current_path, + ] + + if extra_dir: + # Add extra strategy directory on top of search paths + abs_paths.insert(0, extra_dir) + + for path in abs_paths: + strategy = self._search_strategy(path, strategy_name) + if strategy: + logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) + return strategy + + raise ImportError( + "Impossible to load Strategy '{}'. This class does not exist" + " or contains Python code errors".format(strategy_name) + ) + + @staticmethod + def _get_valid_strategies(module_path: str, strategy_name: str) -> Optional[Type[IStrategy]]: + """ + Returns a list of all possible strategies for the given module_path + :param module_path: absolute path to the module + :param strategy_name: Class name of the strategy + :return: Tuple with (name, class) or None + """ + + # Generate spec based on absolute path + spec = importlib.util.spec_from_file_location('user_data.strategies', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + valid_strategies_gen = ( + obj for name, obj in inspect.getmembers(module, inspect.isclass) + if strategy_name == name and IStrategy in obj.__bases__ + ) + return next(valid_strategies_gen, None) + + @staticmethod + def _search_strategy(directory: str, strategy_name: str) -> Optional[IStrategy]: + """ + Search for the strategy_name in the given directory + :param directory: relative or absolute directory path + :return: name of the strategy class + """ + logger.debug('Searching for strategy %s in \'%s\'', strategy_name, directory) + for entry in os.listdir(directory): + # Only consider python files + if not entry.endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + strategy = StrategyResolver._get_valid_strategies( + os.path.abspath(os.path.join(directory, entry)), strategy_name + ) + if strategy: + return strategy() + return None diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py deleted file mode 100644 index 0faa64a05..000000000 --- a/freqtrade/strategy/strategy.py +++ /dev/null @@ -1,169 +0,0 @@ -# pragma pylint: disable=attribute-defined-outside-init - -""" -This module load custom strategies -""" -import importlib -import logging -import os -import sys -from collections import OrderedDict - -from pandas import DataFrame - -from freqtrade.constants import Constants -from freqtrade.strategy.interface import IStrategy - -sys.path.insert(0, r'../../user_data/strategies') - - -logger = logging.getLogger(__name__) - - -class Strategy(object): - """ - This class contains all the logic to load custom strategy class - """ - def __init__(self, config: dict = {}) -> None: - """ - Load the custom class from config parameter - :param config: - :return: - """ - # Verify the strategy is in the configuration, otherwise fallback to the default strategy - if 'strategy' in config: - strategy = config['strategy'] - else: - strategy = Constants.DEFAULT_STRATEGY - - # Load the strategy - self._load_strategy(strategy) - - # Set attributes - # Check if we need to override configuration - if 'minimal_roi' in config: - self.custom_strategy.minimal_roi = config['minimal_roi'] - logger.info("Override strategy \'minimal_roi\' with value in config file.") - - if 'stoploss' in config: - self.custom_strategy.stoploss = config['stoploss'] - logger.info( - "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] - ) - - if 'ticker_interval' in config: - self.custom_strategy.ticker_interval = config['ticker_interval'] - logger.info( - "Override strategy \'ticker_interval\' with value in config file: %s.", - config['ticker_interval'] - ) - - # Minimal ROI designed for the strategy - self.minimal_roi = OrderedDict(sorted( - {int(key): value for (key, value) in self.custom_strategy.minimal_roi.items()}.items(), - key=lambda t: t[0])) # sort after converting to number - - # Optimal stoploss designed for the strategy - self.stoploss = float(self.custom_strategy.stoploss) - - self.ticker_interval = self.custom_strategy.ticker_interval - - def _load_strategy(self, strategy_name: str) -> None: - """ - Search and load the custom strategy. If no strategy found, fallback on the default strategy - Set the object into self.custom_strategy - :param strategy_name: name of the module to import - :return: None - """ - - try: - # Start by sanitizing the file name (remove any extensions) - strategy_name = self._sanitize_module_name(filename=strategy_name) - - # Search where can be the strategy file - path = self._search_strategy(filename=strategy_name) - - # Load the strategy - self.custom_strategy = self._load_class(path + strategy_name) - - # Fallback to the default strategy - except (ImportError, TypeError) as error: - logger.error( - "Impossible to load Strategy 'user_data/strategies/%s.py'. This file does not exist" - " or contains Python code errors", - strategy_name - ) - logger.error( - "The error is:\n%s.", - error - ) - - def _load_class(self, filename: str) -> IStrategy: - """ - Import a strategy as a module - :param filename: path to the strategy (path from freqtrade/strategy/) - :return: return the strategy class - """ - module = importlib.import_module(filename, __package__) - custom_strategy = getattr(module, module.class_name) - - logger.info("Load strategy class: %s (%s.py)", module.class_name, filename) - return custom_strategy() - - @staticmethod - def _sanitize_module_name(filename: str) -> str: - """ - Remove any extension from filename - :param filename: filename to sanatize - :return: return the filename without extensions - """ - filename = os.path.basename(filename) - filename = os.path.splitext(filename)[0] - return filename - - @staticmethod - def _search_strategy(filename: str) -> str: - """ - Search for the Strategy file in different folder - 1. search into the user_data/strategies folder - 2. search into the freqtrade/strategy folder - 3. if nothing found, return None - :param strategy_name: module name to search - :return: module path where is the strategy - """ - pwd = os.path.dirname(os.path.realpath(__file__)) + '/' - user_data = os.path.join(pwd, '..', '..', 'user_data', 'strategies', filename + '.py') - strategy_folder = os.path.join(pwd, filename + '.py') - - path = None - if os.path.isfile(user_data): - path = 'user_data.strategies.' - elif os.path.isfile(strategy_folder): - path = '.' - - return path - - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Populate indicators that will be used in the Buy and Sell strategy - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() - :return: a Dataframe with all mandatory indicators for the strategies - """ - return self.custom_strategy.populate_indicators(dataframe) - - def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - :return: - """ - return self.custom_strategy.populate_buy_trend(dataframe) - - def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - return self.custom_strategy.populate_sell_trend(dataframe) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 060073771..bb42bcff9 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -12,7 +12,7 @@ from sqlalchemy import create_engine from telegram import Chat, Message, Update from freqtrade.analyze import Analyze -from freqtrade.constants import Constants +from freqtrade import constants from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -87,7 +87,7 @@ def default_conf(): "initial_state": "running", "loglevel": logging.DEBUG } - validate(configuration, Constants.CONF_SCHEMA) + validate(configuration, constants.CONF_SCHEMA) return configuration @@ -207,7 +207,7 @@ def markets_empty(): return MagicMock(return_value=[]) -@pytest.fixture +@pytest.fixture(scope='function') def limit_buy_order(): return { 'id': 'mocked_limit_buy', @@ -302,7 +302,7 @@ def ticker_history(): 0.05874751, ], [ - 1511686800, + 1511686800000, 8.891e-05, 8.893e-05, 8.875e-05, @@ -498,3 +498,90 @@ def result(): # that inserts a trade of some type and open-status # return the open-order-id # See tests in rpc/main that could use this + + +@pytest.fixture(scope="function") +def trades_for_order(): + return [{'info': {'id': 34567, + 'orderId': 123456, + 'price': '0.24544100', + 'qty': '8.00000000', + 'commission': '0.00800000', + 'commissionAsset': 'LTC', + 'time': 1521663363189, + 'isBuyer': True, + 'isMaker': False, + 'isBestMatch': True}, + 'timestamp': 1521663363189, + 'datetime': '2018-03-21T20:16:03.189Z', + 'symbol': 'LTC/ETH', + 'id': '34567', + 'order': '123456', + 'type': None, + 'side': 'buy', + 'price': 0.245441, + 'cost': 1.963528, + 'amount': 8.0, + 'fee': {'cost': 0.008, 'currency': 'LTC'}}] + + +@pytest.fixture(scope="function") +def trades_for_order2(): + return [{'info': {'id': 34567, + 'orderId': 123456, + 'price': '0.24544100', + 'qty': '8.00000000', + 'commission': '0.00800000', + 'commissionAsset': 'LTC', + 'time': 1521663363189, + 'isBuyer': True, + 'isMaker': False, + 'isBestMatch': True}, + 'timestamp': 1521663363189, + 'datetime': '2018-03-21T20:16:03.189Z', + 'symbol': 'LTC/ETH', + 'id': '34567', + 'order': '123456', + 'type': None, + 'side': 'buy', + 'price': 0.245441, + 'cost': 1.963528, + 'amount': 4.0, + 'fee': {'cost': 0.004, 'currency': 'LTC'}}, + {'info': {'id': 34567, + 'orderId': 123456, + 'price': '0.24544100', + 'qty': '8.00000000', + 'commission': '0.00800000', + 'commissionAsset': 'LTC', + 'time': 1521663363189, + 'isBuyer': True, + 'isMaker': False, + 'isBestMatch': True}, + 'timestamp': 1521663363189, + 'datetime': '2018-03-21T20:16:03.189Z', + 'symbol': 'LTC/ETH', + 'id': '34567', + 'order': '123456', + 'type': None, + 'side': 'buy', + 'price': 0.245441, + 'cost': 1.963528, + 'amount': 4.0, + 'fee': {'cost': 0.004, 'currency': 'LTC'}}] + + +@pytest.fixture +def buy_order_fee(): + return { + 'id': 'mocked_limit_buy_old', + 'type': 'limit', + 'side': 'buy', + 'pair': 'mocked', + 'datetime': str(arrow.utcnow().shift(minutes=-601).datetime), + 'price': 0.245441, + 'amount': 8.0, + 'remaining': 90.99181073, + 'status': 'closed', + 'fee': None + } diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index e194cb072..a506a8416 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -4,14 +4,15 @@ import logging from copy import deepcopy from random import randint from unittest.mock import MagicMock, PropertyMock -import ccxt +import ccxt import pytest -from freqtrade import OperationalException, DependencyException, NetworkException -from freqtrade.exchange import init, validate_pairs, buy, sell, get_balance, get_balances, \ - get_ticker, get_ticker_history, cancel_order, get_name, get_fee, get_id, get_pair_detail_url import freqtrade.exchange as exchange +from freqtrade import OperationalException, DependencyException, TemporaryError +from freqtrade.exchange import (init, validate_pairs, buy, sell, get_balance, get_balances, + get_ticker, get_ticker_history, cancel_order, get_name, get_fee, + get_id, get_pair_detail_url, get_amount_lots) from freqtrade.tests.conftest import log_has API_INIT = False @@ -148,7 +149,7 @@ def test_buy_prod(default_conf, mocker): mocker.patch('freqtrade.exchange._API', api_mock) buy(pair='ETH/BTC', rate=200, amount=1) - with pytest.raises(NetworkException): + with pytest.raises(TemporaryError): api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.NetworkError) mocker.patch('freqtrade.exchange._API', api_mock) buy(pair='ETH/BTC', rate=200, amount=1) @@ -198,7 +199,7 @@ def test_sell_prod(default_conf, mocker): mocker.patch('freqtrade.exchange._API', api_mock) sell(pair='ETH/BTC', rate=200, amount=1) - with pytest.raises(NetworkException): + with pytest.raises(TemporaryError): api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.NetworkError) mocker.patch('freqtrade.exchange._API', api_mock) sell(pair='ETH/BTC', rate=200, amount=1) @@ -262,15 +263,17 @@ def test_get_balances_prod(default_conf, mocker): assert get_balances()['1ST']['total'] == 10.0 assert get_balances()['1ST']['used'] == 0.0 - with pytest.raises(NetworkException): + with pytest.raises(TemporaryError): api_mock.fetch_balance = MagicMock(side_effect=ccxt.NetworkError) mocker.patch('freqtrade.exchange._API', api_mock) get_balances() + assert api_mock.fetch_balance.call_count == exchange.API_RETRY_COUNT + 1 with pytest.raises(OperationalException): api_mock.fetch_balance = MagicMock(side_effect=ccxt.BaseError) mocker.patch('freqtrade.exchange._API', api_mock) get_balances() + assert api_mock.fetch_balance.call_count == 1 # This test is somewhat redundant with @@ -310,7 +313,7 @@ def test_get_ticker(default_conf, mocker): assert ticker['bid'] == 0.5 assert ticker['ask'] == 1 - with pytest.raises(OperationalException): # test retrier + with pytest.raises(TemporaryError): # test retrier api_mock.fetch_ticker = MagicMock(side_effect=ccxt.NetworkError) mocker.patch('freqtrade.exchange._API', api_mock) get_ticker(pair='ETH/BTC', refresh=True) @@ -377,7 +380,7 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][4] == 9 assert ticks[0][5] == 10 - with pytest.raises(OperationalException): # test retrier + with pytest.raises(TemporaryError): # test retrier api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NetworkError) mocker.patch('freqtrade.exchange._API', api_mock) # new symbol to get around cache @@ -406,20 +409,23 @@ def test_cancel_order(default_conf, mocker): mocker.patch('freqtrade.exchange._API', api_mock) assert cancel_order(order_id='_', pair='TKN/BTC') == 123 - with pytest.raises(NetworkException): + with pytest.raises(TemporaryError): api_mock.cancel_order = MagicMock(side_effect=ccxt.NetworkError) mocker.patch('freqtrade.exchange._API', api_mock) cancel_order(order_id='_', pair='TKN/BTC') + assert api_mock.cancel_order.call_count == exchange.API_RETRY_COUNT + 1 with pytest.raises(DependencyException): api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder) mocker.patch('freqtrade.exchange._API', api_mock) cancel_order(order_id='_', pair='TKN/BTC') + assert api_mock.cancel_order.call_count == exchange.API_RETRY_COUNT + 1 with pytest.raises(OperationalException): api_mock.cancel_order = MagicMock(side_effect=ccxt.BaseError) mocker.patch('freqtrade.exchange._API', api_mock) cancel_order(order_id='_', pair='TKN/BTC') + assert api_mock.cancel_order.call_count == 1 def test_get_order(default_conf, mocker): @@ -438,20 +444,23 @@ def test_get_order(default_conf, mocker): mocker.patch('freqtrade.exchange._API', api_mock) assert exchange.get_order('X', 'TKN/BTC') == 456 - with pytest.raises(NetworkException): + with pytest.raises(TemporaryError): api_mock.fetch_order = MagicMock(side_effect=ccxt.NetworkError) mocker.patch('freqtrade.exchange._API', api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') + assert api_mock.fetch_order.call_count == exchange.API_RETRY_COUNT + 1 with pytest.raises(DependencyException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder) mocker.patch('freqtrade.exchange._API', api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') + assert api_mock.fetch_order.call_count == exchange.API_RETRY_COUNT + 1 with pytest.raises(OperationalException): api_mock.fetch_order = MagicMock(side_effect=ccxt.BaseError) mocker.patch('freqtrade.exchange._API', api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') + assert api_mock.fetch_order.call_count == 1 def test_get_name(default_conf, mocker): @@ -508,3 +517,10 @@ def test_get_fee(default_conf, mocker): }) mocker.patch('freqtrade.exchange._API', api_mock) assert get_fee() == 0.025 + + +def test_get_amount_lots(default_conf, mocker): + api_mock = MagicMock() + api_mock.amount_to_lots = MagicMock(return_value=1.0) + mocker.patch('freqtrade.exchange._API', api_mock) + assert get_amount_lots('LTC/BTC', 1.54) == 1 diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f5709154b..f17a0115e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -6,7 +6,6 @@ import random from copy import deepcopy from typing import List from unittest.mock import MagicMock -import pytest import numpy as np import pandas as pd @@ -18,19 +17,6 @@ from freqtrade.arguments import Arguments from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration from freqtrade.tests.conftest import log_has -# Avoid to reinit the same object again and again -_BACKTESTING = None -_BACKTESTING_INITIALIZED = False - - -@pytest.fixture(scope='function') -def init_backtesting(default_conf, mocker): - global _BACKTESTING_INITIALIZED, _BACKTESTING - if not _BACKTESTING_INITIALIZED: - mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - _BACKTESTING = Backtesting(default_conf) - _BACKTESTING_INITIALIZED = True - def get_args(args) -> List[str]: return Arguments(args, '').get_parsed_arg() @@ -96,8 +82,9 @@ def load_data_test(what): return data -def simple_backtest(config, contour, num_results) -> None: - backtesting = _BACKTESTING +def simple_backtest(config, contour, num_results, mocker) -> None: + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + backtesting = Backtesting(config) data = load_data_test(contour) processed = backtesting.tickerdata_to_dataframe(data) @@ -128,12 +115,14 @@ def _load_pair_as_ticks(pair, tickfreq): # FIX: fixturize this? -def _make_backtest_conf(conf=None, pair='UNITTEST/BTC', record=None): +def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): data = optimize.load_data(None, ticker_interval='8m', pairs=[pair]) data = trim_dictlist(data, -200) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + backtesting = Backtesting(conf) return { 'stake_amount': conf['stake_amount'], - 'processed': _BACKTESTING.tickerdata_to_dataframe(data), + 'processed': backtesting.tickerdata_to_dataframe(data), 'max_open_trades': 10, 'realistic': True, 'record': record @@ -169,21 +158,6 @@ def _trend_alternate(dataframe=None): return dataframe -def _run_backtest_1(fun, backtest_conf): - # strategy is a global (hidden as a singleton), so we - # emulate strategy being pure, by override/restore here - # if we dont do this, the override in strategy will carry over - # to other tests - old_buy = _BACKTESTING.populate_buy_trend - old_sell = _BACKTESTING.populate_sell_trend - _BACKTESTING.populate_buy_trend = fun # Override - _BACKTESTING.populate_sell_trend = fun # Override - results = _BACKTESTING.backtest(backtest_conf) - _BACKTESTING.populate_buy_trend = old_buy # restore override - _BACKTESTING.populate_sell_trend = old_sell # restore override - return results - - # Unit tests def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: """ @@ -195,7 +169,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', 'backtesting' ] @@ -236,7 +210,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', '--datadir', '/foo/bar', 'backtesting', '--ticker-interval', '1m', @@ -287,19 +261,20 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non ) -def test_start(mocker, init_backtesting, fee, default_conf, caplog) -> None: +def test_start(mocker, fee, default_conf, caplog) -> None: """ Test start() function """ start_mock = MagicMock() mocker.patch('freqtrade.exchange.get_fee', fee) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch('freqtrade.optimize.backtesting.Backtesting.start', start_mock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', 'backtesting' ] args = get_args(args) @@ -342,16 +317,16 @@ def test_backtesting_init(mocker, default_conf) -> None: assert callable(backtesting.populate_sell_trend) -def test_tickerdata_to_dataframe(init_backtesting, default_conf) -> None: +def test_tickerdata_to_dataframe(default_conf, mocker) -> None: """ Test Backtesting.tickerdata_to_dataframe() method """ - + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) timerange = ((None, 'line'), None, -100) tick = optimize.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': tick} - backtesting = _BACKTESTING + backtesting = Backtesting(default_conf) data = backtesting.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 100 @@ -361,11 +336,12 @@ def test_tickerdata_to_dataframe(init_backtesting, default_conf) -> None: assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC']) -def test_get_timeframe(init_backtesting) -> None: +def test_get_timeframe(default_conf, mocker) -> None: """ Test Backtesting.get_timeframe() method """ - backtesting = _BACKTESTING + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + backtesting = Backtesting(default_conf) data = backtesting.tickerdata_to_dataframe( optimize.load_data( @@ -379,11 +355,12 @@ def test_get_timeframe(init_backtesting) -> None: assert max_date.isoformat() == '2017-11-14T22:59:00+00:00' -def test_generate_text_table(init_backtesting): +def test_generate_text_table(default_conf, mocker): """ Test Backtesting.generate_text_table() method """ - backtesting = _BACKTESTING + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + backtesting = Backtesting(default_conf) results = pd.DataFrame( { @@ -451,13 +428,13 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: assert log_has(line, caplog.record_tuples) -def test_backtest(init_backtesting, default_conf, fee, mocker) -> None: +def test_backtest(default_conf, fee, mocker) -> None: """ Test Backtesting.backtest() method """ mocker.patch('freqtrade.exchange.get_fee', fee) - - backtesting = _BACKTESTING + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + backtesting = Backtesting(default_conf) data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) @@ -472,13 +449,13 @@ def test_backtest(init_backtesting, default_conf, fee, mocker) -> None: assert not results.empty -def test_backtest_1min_ticker_interval(init_backtesting, default_conf, fee, mocker) -> None: +def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: """ Test Backtesting.backtest() method with 1 min ticker """ mocker.patch('freqtrade.exchange.get_fee', fee) - - backtesting = _BACKTESTING + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + backtesting = Backtesting(default_conf) # Run a backtesting for an exiting 5min ticker_interval data = optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) @@ -494,11 +471,12 @@ def test_backtest_1min_ticker_interval(init_backtesting, default_conf, fee, mock assert not results.empty -def test_processed(init_backtesting) -> None: +def test_processed(default_conf, mocker) -> None: """ Test Backtesting.backtest() method with offline data """ - backtesting = _BACKTESTING + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + backtesting = Backtesting(default_conf) dict_of_tickerrows = load_data_test('raise') dataframes = backtesting.tickerdata_to_dataframe(dict_of_tickerrows) @@ -510,69 +488,90 @@ def test_processed(init_backtesting) -> None: assert col in cols -def test_backtest_pricecontours(init_backtesting, default_conf, fee, mocker) -> None: +def test_backtest_pricecontours(default_conf, fee, mocker) -> None: mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) tests = [['raise', 17], ['lower', 0], ['sine', 17]] for [contour, numres] in tests: - simple_backtest(default_conf, contour, numres) + simple_backtest(default_conf, contour, numres, mocker) # Test backtest using offline data (testdata directory) -def test_backtest_ticks(init_backtesting, default_conf, fee, mocker): +def test_backtest_ticks(default_conf, fee, mocker): mocker.patch('freqtrade.exchange.get_fee', fee) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) ticks = [1, 5] - fun = _BACKTESTING.populate_buy_trend + fun = Backtesting(default_conf).populate_buy_trend for _ in ticks: - backtest_conf = _make_backtest_conf(conf=default_conf) - results = _run_backtest_1(fun, backtest_conf) + backtest_conf = _make_backtest_conf(mocker, conf=default_conf) + backtesting = Backtesting(default_conf) + backtesting.populate_buy_trend = fun # Override + backtesting.populate_sell_trend = fun # Override + results = backtesting.backtest(backtest_conf) assert not results.empty -def test_backtest_clash_buy_sell(init_backtesting, default_conf): +def test_backtest_clash_buy_sell(mocker, default_conf): # Override the default buy trend function in our default_strategy def fun(dataframe=None): buy_value = 1 sell_value = 1 return _trend(dataframe, buy_value, sell_value) - backtest_conf = _make_backtest_conf(conf=default_conf) - results = _run_backtest_1(fun, backtest_conf) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + backtest_conf = _make_backtest_conf(mocker, conf=default_conf) + backtesting = Backtesting(default_conf) + backtesting.populate_buy_trend = fun # Override + backtesting.populate_sell_trend = fun # Override + results = backtesting.backtest(backtest_conf) assert results.empty -def test_backtest_only_sell(init_backtesting, default_conf): +def test_backtest_only_sell(mocker, default_conf): # Override the default buy trend function in our default_strategy def fun(dataframe=None): buy_value = 0 sell_value = 1 return _trend(dataframe, buy_value, sell_value) - backtest_conf = _make_backtest_conf(conf=default_conf) - results = _run_backtest_1(fun, backtest_conf) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + backtest_conf = _make_backtest_conf(mocker, conf=default_conf) + backtesting = Backtesting(default_conf) + backtesting.populate_buy_trend = fun # Override + backtesting.populate_sell_trend = fun # Override + results = backtesting.backtest(backtest_conf) assert results.empty -def test_backtest_alternate_buy_sell(init_backtesting, default_conf, fee, mocker): +def test_backtest_alternate_buy_sell(default_conf, fee, mocker): mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) - backtest_conf = _make_backtest_conf(conf=default_conf, pair='UNITTEST/BTC') - results = _run_backtest_1(_trend_alternate, backtest_conf) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC') + backtesting = Backtesting(default_conf) + backtesting.populate_buy_trend = _trend_alternate # Override + backtesting.populate_sell_trend = _trend_alternate # Override + results = backtesting.backtest(backtest_conf) assert len(results) == 3 -def test_backtest_record(init_backtesting, default_conf, fee, mocker): +def test_backtest_record(default_conf, fee, mocker): names = [] records = [] + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) mocker.patch( 'freqtrade.optimize.backtesting.file_dump_json', new=lambda n, r: (names.append(n), records.append(r)) ) backtest_conf = _make_backtest_conf( + mocker, conf=default_conf, pair='UNITTEST/BTC', record="trades" ) - results = _run_backtest_1(_trend_alternate, backtest_conf) + backtesting = Backtesting(default_conf) + backtesting.populate_buy_trend = _trend_alternate # Override + backtesting.populate_sell_trend = _trend_alternate # Override + results = backtesting.backtest(backtest_conf) assert len(results) == 3 # Assert file_dump_json was only called once assert names == ['backtest-result.json'] @@ -595,7 +594,7 @@ def test_backtest_record(init_backtesting, default_conf, fee, mocker): assert dur > 0 -def test_backtest_start_live(init_backtesting, default_conf, mocker, caplog): +def test_backtest_start_live(default_conf, mocker, caplog): conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] mocker.patch('freqtrade.exchange.get_ticker_history', @@ -613,12 +612,12 @@ def test_backtest_start_live(init_backtesting, default_conf, mocker, caplog): args.live = True args.datadir = None args.export = None - args.strategy = 'default_strategy' + args.strategy = 'DefaultStrategy' args.timerange = '-100' # needed due to MagicMock malleability args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', 'backtesting', '--ticker-interval', '1m', '--live', diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 4916c1040..68fc99955 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -3,17 +3,16 @@ import os import signal from copy import deepcopy from unittest.mock import MagicMock -import pytest import pandas as pd +import pytest from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt, start -from freqtrade.strategy.strategy import Strategy +from freqtrade.strategy.resolver import StrategyResolver from freqtrade.tests.conftest import log_has from freqtrade.tests.optimize.test_backtesting import get_args - # Avoid to reinit the same object again and again _HYPEROPT_INITIALIZED = False _HYPEROPT = None @@ -71,12 +70,12 @@ def test_start(mocker, default_conf, caplog) -> None: args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', 'hyperopt', '--epochs', '5' ] args = get_args(args) - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) start(args) import pprint @@ -94,7 +93,7 @@ def test_loss_calculation_prefer_correct_trade_count(init_hyperopt) -> None: Test Hyperopt.calculate_loss() """ hyperopt = _HYPEROPT - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20) over = hyperopt.calculate_loss(1, hyperopt.target_trades + 100, 20) @@ -124,7 +123,7 @@ def test_loss_calculation_has_limited_profit(init_hyperopt) -> None: assert under > correct -def test_log_results_if_loss_improves(init_hyperopt, capsys) -> None: +def test_log_results_if_loss_improves(capsys) -> None: hyperopt = _HYPEROPT hyperopt.current_best_loss = 2 hyperopt.log_results( @@ -186,7 +185,7 @@ def test_fmin_best_results(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = create_trials(mocker) hyperopt.tickerdata_to_dataframe = MagicMock() @@ -231,7 +230,7 @@ def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = create_trials(mocker) hyperopt.tickerdata_to_dataframe = MagicMock() @@ -274,7 +273,7 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.exchange.validate_pairs', MagicMock()) - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = trials hyperopt.tickerdata_to_dataframe = MagicMock() diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index f0c429792..5331df5e7 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -291,10 +291,12 @@ def test_download_backtesting_testdata2(mocker) -> None: [1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839], [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] - mocker.patch('freqtrade.misc.file_dump_json', return_value=None) + json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick) - assert download_backtesting_testdata(None, pair="UNITTEST/BTC", tick_interval='1m') - assert download_backtesting_testdata(None, pair="UNITTEST/BTC", tick_interval='3m') + + download_backtesting_testdata(None, pair="UNITTEST/BTC", interval='1m') + download_backtesting_testdata(None, pair="UNITTEST/BTC", interval='3m') + assert json_dump_mock.call_count == 2 def test_load_tickerdata_file() -> None: diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index cb6cbd4ca..900fc2234 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -1,10 +1,16 @@ +import json + +import pytest from pandas import DataFrame -from freqtrade.strategy.default_strategy import DefaultStrategy, class_name +from freqtrade.analyze import Analyze +from freqtrade.strategy.default_strategy import DefaultStrategy -def test_default_strategy_class_name(): - assert class_name == DefaultStrategy.__name__ +@pytest.fixture +def result(): + with open('freqtrade/tests/testdata/ETH_BTC-1m.json') as data_file: + return Analyze.parse_ticker_dataframe(json.load(data_file)) def test_default_strategy_structure(): diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index a364a59fe..244910790 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,89 +1,85 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging +import os -from freqtrade.strategy.strategy import Strategy +import pytest - -def test_sanitize_module_name(): - assert Strategy._sanitize_module_name('default_strategy') == 'default_strategy' - assert Strategy._sanitize_module_name('default_strategy.py') == 'default_strategy' - assert Strategy._sanitize_module_name('../default_strategy.py') == 'default_strategy' - assert Strategy._sanitize_module_name('../default_strategy') == 'default_strategy' - assert Strategy._sanitize_module_name('.default_strategy') == '.default_strategy' - assert Strategy._sanitize_module_name('foo-bar') == 'foo-bar' - assert Strategy._sanitize_module_name('foo/bar') == 'bar' +from freqtrade.strategy.interface import IStrategy +from freqtrade.strategy.resolver import StrategyResolver def test_search_strategy(): - assert Strategy._search_strategy('default_strategy') == '.' - assert Strategy._search_strategy('test_strategy') == 'user_data.strategies.' - assert Strategy._search_strategy('super_duper') is None - - -def test_strategy_structure(): - assert hasattr(Strategy, 'populate_indicators') - assert hasattr(Strategy, 'populate_buy_trend') - assert hasattr(Strategy, 'populate_sell_trend') + default_location = os.path.join(os.path.dirname( + os.path.realpath(__file__)), '..', '..', 'strategy' + ) + assert isinstance( + StrategyResolver._search_strategy(default_location, 'DefaultStrategy'), IStrategy + ) + assert StrategyResolver._search_strategy(default_location, 'NotFoundStrategy') is None def test_load_strategy(result): - strategy = Strategy() - - assert not hasattr(Strategy, 'custom_strategy') - strategy._load_strategy('test_strategy') - - assert not hasattr(Strategy, 'custom_strategy') - - assert hasattr(strategy.custom_strategy, 'populate_indicators') - assert 'adx' in strategy.populate_indicators(result) + resolver = StrategyResolver() + resolver._load_strategy('TestStrategy') + assert hasattr(resolver.strategy, 'populate_indicators') + assert 'adx' in resolver.strategy.populate_indicators(result) -def test_load_not_found_strategy(caplog): - strategy = Strategy() +def test_load_strategy_custom_directory(result): + resolver = StrategyResolver() + extra_dir = os.path.join('some', 'path') + with pytest.raises( + FileNotFoundError, + match=r".*No such file or directory: '{}'".format(extra_dir)): + resolver._load_strategy('TestStrategy', extra_dir) - assert not hasattr(Strategy, 'custom_strategy') - strategy._load_strategy('NotFoundStrategy') + assert hasattr(resolver.strategy, 'populate_indicators') + assert 'adx' in resolver.strategy.populate_indicators(result) - error_msg = "Impossible to load Strategy 'user_data/strategies/{}.py'. This file does not " \ - "exist or contains Python code errors".format('NotFoundStrategy') - assert ('freqtrade.strategy.strategy', logging.ERROR, error_msg) in caplog.record_tuples + +def test_load_not_found_strategy(): + strategy = StrategyResolver() + with pytest.raises(ImportError, + match=r'Impossible to load Strategy \'NotFoundStrategy\'.' + r' This class does not exist or contains Python code errors'): + strategy._load_strategy('NotFoundStrategy') def test_strategy(result): - strategy = Strategy({'strategy': 'default_strategy'}) + resolver = StrategyResolver({'strategy': 'DefaultStrategy'}) - assert hasattr(strategy.custom_strategy, 'minimal_roi') - assert strategy.minimal_roi[0] == 0.04 + assert hasattr(resolver.strategy, 'minimal_roi') + assert resolver.strategy.minimal_roi[0] == 0.04 - assert hasattr(strategy.custom_strategy, 'stoploss') - assert strategy.stoploss == -0.10 + assert hasattr(resolver.strategy, 'stoploss') + assert resolver.strategy.stoploss == -0.10 - assert hasattr(strategy.custom_strategy, 'populate_indicators') - assert 'adx' in strategy.populate_indicators(result) + assert hasattr(resolver.strategy, 'populate_indicators') + assert 'adx' in resolver.strategy.populate_indicators(result) - assert hasattr(strategy.custom_strategy, 'populate_buy_trend') - dataframe = strategy.populate_buy_trend(strategy.populate_indicators(result)) + assert hasattr(resolver.strategy, 'populate_buy_trend') + dataframe = resolver.strategy.populate_buy_trend(resolver.strategy.populate_indicators(result)) assert 'buy' in dataframe.columns - assert hasattr(strategy.custom_strategy, 'populate_sell_trend') - dataframe = strategy.populate_sell_trend(strategy.populate_indicators(result)) + assert hasattr(resolver.strategy, 'populate_sell_trend') + dataframe = resolver.strategy.populate_sell_trend(resolver.strategy.populate_indicators(result)) assert 'sell' in dataframe.columns def test_strategy_override_minimal_roi(caplog): caplog.set_level(logging.INFO) config = { - 'strategy': 'default_strategy', + 'strategy': 'DefaultStrategy', 'minimal_roi': { "0": 0.5 } } - strategy = Strategy(config) + resolver = StrategyResolver(config) - assert hasattr(strategy.custom_strategy, 'minimal_roi') - assert strategy.minimal_roi[0] == 0.5 - assert ('freqtrade.strategy.strategy', + assert hasattr(resolver.strategy, 'minimal_roi') + assert resolver.strategy.minimal_roi[0] == 0.5 + assert ('freqtrade.strategy.resolver', logging.INFO, 'Override strategy \'minimal_roi\' with value in config file.' ) in caplog.record_tuples @@ -92,14 +88,14 @@ def test_strategy_override_minimal_roi(caplog): def test_strategy_override_stoploss(caplog): caplog.set_level(logging.INFO) config = { - 'strategy': 'default_strategy', + 'strategy': 'DefaultStrategy', 'stoploss': -0.5 } - strategy = Strategy(config) + resolver = StrategyResolver(config) - assert hasattr(strategy.custom_strategy, 'stoploss') - assert strategy.stoploss == -0.5 - assert ('freqtrade.strategy.strategy', + assert hasattr(resolver.strategy, 'stoploss') + assert resolver.strategy.stoploss == -0.5 + assert ('freqtrade.strategy.resolver', logging.INFO, 'Override strategy \'stoploss\' with value in config file: -0.5.' ) in caplog.record_tuples @@ -109,34 +105,14 @@ def test_strategy_override_ticker_interval(caplog): caplog.set_level(logging.INFO) config = { - 'strategy': 'default_strategy', + 'strategy': 'DefaultStrategy', 'ticker_interval': 60 } - strategy = Strategy(config) + resolver = StrategyResolver(config) - assert hasattr(strategy.custom_strategy, 'ticker_interval') - assert strategy.ticker_interval == 60 - assert ('freqtrade.strategy.strategy', + assert hasattr(resolver.strategy, 'ticker_interval') + assert resolver.strategy.ticker_interval == 60 + assert ('freqtrade.strategy.resolver', logging.INFO, 'Override strategy \'ticker_interval\' with value in config file: 60.' ) in caplog.record_tuples - - -def test_strategy_fallback_default_strategy(): - strategy = Strategy() - strategy.logger = logging.getLogger(__name__) - - assert not hasattr(Strategy, 'custom_strategy') - strategy._load_strategy('../../super_duper') - assert not hasattr(Strategy, 'custom_strategy') - - -def test_strategy_singleton(): - strategy1 = Strategy({'strategy': 'default_strategy'}) - - assert hasattr(strategy1.custom_strategy, 'minimal_roi') - assert strategy1.minimal_roi[0] == 0.04 - - strategy2 = Strategy() - assert hasattr(strategy2.custom_strategy, 'minimal_roi') - assert strategy2.minimal_roi[0] == 0.04 diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 8d521150d..01033ce7d 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -16,7 +16,7 @@ from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import log_has # Avoid to reinit the same object again and again -_ANALYZE = Analyze({'strategy': 'default_strategy'}) +_ANALYZE = Analyze({'strategy': 'DefaultStrategy'}) def test_signaltype_object() -> None: diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 756f48778..279ace0dc 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -71,6 +71,26 @@ def test_parse_args_invalid() -> None: Arguments(['-c'], '').get_parsed_arg() +def test_parse_args_strategy() -> None: + args = Arguments(['--strategy', 'SomeStrategy'], '').get_parsed_arg() + assert args.strategy == 'SomeStrategy' + + +def test_parse_args_strategy_invalid() -> None: + with pytest.raises(SystemExit, match=r'2'): + Arguments(['--strategy'], '').get_parsed_arg() + + +def test_parse_args_strategy_path() -> None: + args = Arguments(['--strategy-path', '/some/path'], '').get_parsed_arg() + assert args.strategy_path == '/some/path' + + +def test_parse_args_strategy_path_invalid() -> None: + with pytest.raises(SystemExit, match=r'2'): + Arguments(['--strategy-path'], '').get_parsed_arg() + + def test_parse_args_dynamic_whitelist() -> None: args = Arguments(['--dynamic-whitelist'], '').get_parsed_arg() assert args.dynamic_whitelist == 20 diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index c45fa94c8..da7a042f5 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -99,8 +99,8 @@ def test_load_config(default_conf, mocker) -> None: configuration = Configuration(args) validated_conf = configuration.load_config() - assert 'strategy' in validated_conf - assert validated_conf['strategy'] == 'default_strategy' + assert validated_conf.get('strategy') == 'DefaultStrategy' + assert validated_conf.get('strategy_path') is None assert 'dynamic_whitelist' not in validated_conf assert 'dry_run_db' not in validated_conf @@ -115,20 +115,40 @@ def test_load_config_with_params(default_conf, mocker) -> None: args = [ '--dynamic-whitelist', '10', - '--strategy', 'test_strategy', - '--dry-run-db' + '--strategy', 'TestStrategy', + '--strategy-path', '/some/path', + '--dry-run-db', ] args = Arguments(args, '').get_parsed_arg() configuration = Configuration(args) validated_conf = configuration.load_config() - assert 'dynamic_whitelist' in validated_conf - assert validated_conf['dynamic_whitelist'] == 10 - assert 'strategy' in validated_conf - assert validated_conf['strategy'] == 'test_strategy' - assert 'dry_run_db' in validated_conf - assert validated_conf['dry_run_db'] is True + assert validated_conf.get('dynamic_whitelist') == 10 + assert validated_conf.get('strategy') == 'TestStrategy' + assert validated_conf.get('strategy_path') == '/some/path' + assert validated_conf.get('dry_run_db') is True + + +def test_load_custom_strategy(default_conf, mocker) -> None: + """ + Test Configuration.load_config() without any cli params + """ + custom_conf = deepcopy(default_conf) + custom_conf.update({ + 'strategy': 'CustomStrategy', + 'strategy_path': '/tmp/strategies', + }) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(custom_conf) + )) + + args = Arguments([], '').get_parsed_arg() + configuration = Configuration(args) + validated_conf = configuration.load_config() + + assert validated_conf.get('strategy') == 'CustomStrategy' + assert validated_conf.get('strategy_path') == '/tmp/strategies' def test_show_info(default_conf, mocker, caplog) -> None: @@ -141,7 +161,7 @@ def test_show_info(default_conf, mocker, caplog) -> None: args = [ '--dynamic-whitelist', '10', - '--strategy', 'test_strategy', + '--strategy', 'TestStrategy', '--dry-run-db' ] args = Arguments(args, '').get_parsed_arg() @@ -185,7 +205,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', 'backtesting' ] @@ -229,7 +249,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', '--datadir', '/foo/bar', 'backtesting', '--ticker-interval', '1m', diff --git a/freqtrade/tests/test_constants.py b/freqtrade/tests/test_constants.py index 6d544502f..541c6e533 100644 --- a/freqtrade/tests/test_constants.py +++ b/freqtrade/tests/test_constants.py @@ -2,25 +2,24 @@ Unit test file for constants.py """ -from freqtrade.constants import Constants +from freqtrade import constants def test_constant_object() -> None: """ Test the Constants object has the mandatory Constants """ - assert hasattr(Constants, 'CONF_SCHEMA') - assert hasattr(Constants, 'DYNAMIC_WHITELIST') - assert hasattr(Constants, 'PROCESS_THROTTLE_SECS') - assert hasattr(Constants, 'TICKER_INTERVAL') - assert hasattr(Constants, 'HYPEROPT_EPOCH') - assert hasattr(Constants, 'RETRY_TIMEOUT') - assert hasattr(Constants, 'DEFAULT_STRATEGY') + assert hasattr(constants, 'CONF_SCHEMA') + assert hasattr(constants, 'DYNAMIC_WHITELIST') + assert hasattr(constants, 'PROCESS_THROTTLE_SECS') + assert hasattr(constants, 'TICKER_INTERVAL') + assert hasattr(constants, 'HYPEROPT_EPOCH') + assert hasattr(constants, 'RETRY_TIMEOUT') + assert hasattr(constants, 'DEFAULT_STRATEGY') def test_conf_schema() -> None: """ Test the CONF_SCHEMA is from the right type """ - constant = Constants() - assert isinstance(constant.CONF_SCHEMA, dict) + assert isinstance(constants.CONF_SCHEMA, dict) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index 47d89a112..fd461a503 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -4,7 +4,7 @@ import pandas from freqtrade.analyze import Analyze from freqtrade.optimize import load_data -from freqtrade.strategy.strategy import Strategy +from freqtrade.strategy.resolver import StrategyResolver _pairs = ['ETH/BTC'] @@ -15,19 +15,19 @@ def load_dataframe_pair(pairs): assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - analyze = Analyze({'strategy': 'default_strategy'}) + analyze = Analyze({'strategy': 'DefaultStrategy'}) dataframe = analyze.analyze_ticker(dataframe) return dataframe def test_dataframe_load(): - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) dataframe = load_dataframe_pair(_pairs) assert isinstance(dataframe, pandas.core.frame.DataFrame) def test_dataframe_columns_exists(): - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) dataframe = load_dataframe_pair(_pairs) assert 'high' in dataframe.columns assert 'low' in dataframe.columns diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index d307c68aa..ead796aca 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -16,7 +16,7 @@ import pytest import requests from sqlalchemy import create_engine -from freqtrade import DependencyException, OperationalException +from freqtrade import DependencyException, OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.state import State @@ -451,7 +451,7 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non validate_pairs=MagicMock(), get_ticker=ticker, get_markets=markets, - buy=MagicMock(side_effect=requests.exceptions.RequestException) + buy=MagicMock(side_effect=TemporaryError) ) sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) @@ -568,18 +568,30 @@ def test_process_maybe_execute_buy_exception(mocker, default_conf, caplog) -> No log_has('Unable to create trade:', caplog.record_tuples) -def test_process_maybe_execute_sell(mocker, default_conf) -> None: +def test_process_maybe_execute_sell(mocker, default_conf, limit_buy_order, caplog) -> None: """ Test process_maybe_execute_sell() method """ freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) - mocker.patch('freqtrade.freqtradebot.exchange.get_order', return_value=1) + mocker.patch('freqtrade.freqtradebot.exchange.get_order', return_value=limit_buy_order) + mocker.patch('freqtrade.freqtradebot.exchange.get_trades_for_order', return_value=[]) + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', + return_value=limit_buy_order['amount']) trade = MagicMock() trade.open_order_id = '123' + trade.open_fee = 0.001 assert not freqtrade.process_maybe_execute_sell(trade) + # Test amount not modified by fee-logic + assert not log_has('Applying fee to amount for Trade {} from 90.99181073 to 90.81'.format( + trade), caplog.record_tuples) + + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81) + # test amount modified by fee-logic + assert not freqtrade.process_maybe_execute_sell(trade) + trade.is_open = True trade.open_order_id = None # Assert we call handle_trade() if trade is feasible for execution @@ -812,7 +824,8 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe exchange='bittrex', open_order_id='123456789', amount=90.99181073, - fee=0.0, + fee_open=0.0, + fee_close=0.0, stake_amount=1, open_date=arrow.utcnow().shift(minutes=-601).datetime, is_open=True @@ -851,7 +864,8 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, exchange='bittrex', open_order_id='123456789', amount=90.99181073, - fee=0.0, + fee_open=0.0, + fee_close=0.0, stake_amount=1, open_date=arrow.utcnow().shift(hours=-5).datetime, close_date=arrow.utcnow().shift(minutes=-601).datetime, @@ -890,7 +904,8 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old exchange='bittrex', open_order_id='123456789', amount=90.99181073, - fee=0.0, + fee_open=0.0, + fee_close=0.0, stake_amount=1, open_date=arrow.utcnow().shift(minutes=-601).datetime, is_open=True @@ -937,7 +952,8 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - exchange='bittrex', open_order_id='123456789', amount=90.99181073, - fee=0.0, + fee_open=0.0, + fee_close=0.0, stake_amount=1, open_date=arrow.utcnow().shift(minutes=-601).datetime, is_open=True @@ -1299,3 +1315,161 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke trade.update(limit_buy_order) patch_get_signal(mocker, value=(False, True)) assert freqtrade.handle_trade(trade) is True + + +def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, caplog, mocker): + """ + Test get_real_amount - fee in quote currency + """ + + mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order) + + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + amount = sum(x['amount'] for x in trades_for_order) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + open_order_id="123456" + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + # Amount is reduced by "fee" + assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) + assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992) from Trades', + caplog.record_tuples) + + +def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): + """ + Test get_real_amount - fee in quote currency + """ + + mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=[]) + + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + amount = buy_order_fee['amount'] + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + open_order_id="123456" + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + # Amount is reduced by "fee" + assert freqtrade.get_real_amount(trade, buy_order_fee) == amount + assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) failed: myTrade-Dict empty found', + caplog.record_tuples) + + +def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, caplog, mocker): + """ + Test get_real_amount - fees in Stake currency + """ + trades_for_order[0]['fee']['currency'] = 'ETH' + + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order) + amount = sum(x['amount'] for x in trades_for_order) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + open_order_id="123456" + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + # Amount does not change + assert freqtrade.get_real_amount(trade, buy_order_fee) == amount + + +def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mocker): + """ + Test get_real_amount - Fees in BNB + """ + + trades_for_order[0]['fee']['currency'] = 'BNB' + trades_for_order[0]['fee']['cost'] = 0.00094518 + + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order) + amount = sum(x['amount'] for x in trades_for_order) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + open_order_id="123456" + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + # Amount does not change + assert freqtrade.get_real_amount(trade, buy_order_fee) == amount + + +def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, mocker): + """ + Test get_real_amount with split trades (multiple trades for this order) + """ + + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order2) + amount = float(sum(x['amount'] for x in trades_for_order2)) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + open_order_id="123456" + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + # Amount is reduced by "fee" + assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) + assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992) from Trades', + caplog.record_tuples) + + +def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, caplog, mocker): + """ + Test get_real_amount with split trades (multiple trades for this order) + """ + limit_buy_order = deepcopy(buy_order_fee) + limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} + + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order) + amount = float(sum(x['amount'] for x in trades_for_order)) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + open_order_id="123456" + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + # Amount is reduced by "fee" + assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 + assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996) from Order', + caplog.record_tuples) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 44c9db73d..1c6723043 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -42,13 +42,11 @@ def test_datesarray_to_datetimearray(ticker_history): assert date_len == 3 -def test_common_datearray(default_conf, mocker) -> None: +def test_common_datearray(default_conf) -> None: """ Test common_datearray() :return: None """ - mocker.patch('freqtrade.strategy.strategy.Strategy', MagicMock()) - analyze = Analyze(default_conf) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index fa6e15f57..db8a5e9bd 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -126,7 +126,8 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, - fee=fee.return_value, + fee_open=fee.return_value, + fee_close=fee.return_value, exchange='bittrex', ) assert trade.open_order_id is None @@ -154,7 +155,8 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, - fee=fee.return_value, + fee_open=fee.return_value, + fee_close=fee.return_value, exchange='bittrex', ) @@ -177,7 +179,8 @@ def test_calc_close_trade_price_exception(limit_buy_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, - fee=fee.return_value, + fee_open=fee.return_value, + fee_close=fee.return_value, exchange='bittrex', ) @@ -191,7 +194,8 @@ def test_update_open_order(limit_buy_order): trade = Trade( pair='ETH/BTC', stake_amount=1.00, - fee=0.1, + fee_open=0.1, + fee_close=0.1, exchange='bittrex', ) @@ -214,7 +218,8 @@ def test_update_invalid_order(limit_buy_order): trade = Trade( pair='ETH/BTC', stake_amount=1.00, - fee=0.1, + fee_open=0.1, + fee_close=0.1, exchange='bittrex', ) limit_buy_order['type'] = 'invalid' @@ -227,7 +232,8 @@ def test_calc_open_trade_price(limit_buy_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, - fee=fee.return_value, + fee_open=fee.return_value, + fee_close=fee.return_value, exchange='bittrex', ) trade.open_order_id = 'open_trade' @@ -245,7 +251,8 @@ def test_calc_close_trade_price(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, - fee=fee.return_value, + fee_open=fee.return_value, + fee_close=fee.return_value, exchange='bittrex', ) trade.open_order_id = 'close_trade' @@ -267,7 +274,8 @@ def test_calc_profit(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, - fee=fee.return_value, + fee_open=fee.return_value, + fee_close=fee.return_value, exchange='bittrex', ) trade.open_order_id = 'profit_percent' @@ -298,7 +306,8 @@ def test_calc_profit_percent(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', stake_amount=0.001, - fee=fee.return_value, + fee_open=fee.return_value, + fee_close=fee.return_value, exchange='bittrex', ) trade.open_order_id = 'profit_percent' @@ -326,7 +335,8 @@ def test_clean_dry_run_db(default_conf, fee): pair='ETH/BTC', stake_amount=0.001, amount=123.0, - fee=fee.return_value, + fee_open=fee.return_value, + fee_close=fee.return_value, open_rate=0.123, exchange='bittrex', open_order_id='dry_run_buy_12345' @@ -337,7 +347,8 @@ def test_clean_dry_run_db(default_conf, fee): pair='ETC/BTC', stake_amount=0.001, amount=123.0, - fee=fee.return_value, + fee_open=fee.return_value, + fee_close=fee.return_value, open_rate=0.123, exchange='bittrex', open_order_id='dry_run_sell_12345' @@ -349,7 +360,8 @@ def test_clean_dry_run_db(default_conf, fee): pair='ETC/BTC', stake_amount=0.001, amount=123.0, - fee=fee.return_value, + fee_open=fee.return_value, + fee_close=fee.return_value, open_rate=0.123, exchange='bittrex', open_order_id='prod_buy_12345' diff --git a/requirements.txt b/requirements.txt index 6295164bb..4aa22fcd1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ ccxt==1.11.149 -SQLAlchemy==1.2.5 -python-telegram-bot==10.0.1 +SQLAlchemy==1.2.7 +python-telegram-bot==10.0.2 arrow==0.12.1 cachetools==2.0.1 requests==2.18.4 @@ -8,12 +8,12 @@ urllib3==1.22 wrapt==1.10.11 pandas==0.22.0 scikit-learn==0.19.1 -scipy==1.0.0 +scipy==1.0.1 jsonschema==2.6.0 -numpy==1.14.2 +numpy==1.14.3 TA-Lib==0.4.17 -pytest==3.5.0 -pytest-mock==1.7.1 +pytest==3.5.1 +pytest-mock==1.10.0 pytest-cov==2.5.1 hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 136484af2..5e533a030 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -27,7 +27,11 @@ from freqtrade import exchange import freqtrade.optimize as optimize +<<<<<<< HEAD logger = logging.getLogger('freqtrade') +======= +logger = logging.getLogger(__name__) +>>>>>>> bddf009a2b6d0e1a19cca558887ce972e99a6238 def plot_analyzed_dataframe(args: Namespace) -> None: diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 8e121deea..19cf8b11a 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -24,13 +24,20 @@ import plotly.graph_objs as go from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.analyze import Analyze +<<<<<<< HEAD from freqtrade.constants import Constants +======= +>>>>>>> bddf009a2b6d0e1a19cca558887ce972e99a6238 import freqtrade.optimize as optimize import freqtrade.misc as misc +<<<<<<< HEAD logger = logging.getLogger('freqtrade') +======= +logger = logging.getLogger(__name__) +>>>>>>> bddf009a2b6d0e1a19cca558887ce972e99a6238 # data:: [ pair, profit-%, enter, exit, time, duration] diff --git a/setup.sh b/setup.sh index b38f2c645..bdcec7186 100755 --- a/setup.sh +++ b/setup.sh @@ -117,7 +117,7 @@ function config_generator () { -e "s/\"your_exchange_key\"/\"$api_key\"/g" \ -e "s/\"your_exchange_secret\"/\"$api_secret\"/g" \ -e "s/\"your_telegram_token\"/\"$token\"/g" \ - -e "s/\"your_telegram_chat_id\"/\"$chat_id\"/g" + -e "s/\"your_telegram_chat_id\"/\"$chat_id\"/g" \ -e "s/\"dry_run\": false,/\"dry_run\": true,/g" config.json.example > config.json } @@ -205,4 +205,4 @@ plot help ;; esac -exit 0 \ No newline at end of file +exit 0 diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index a164812c4..af28388be 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -10,10 +10,6 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib import numpy # noqa -# Update this variable if you change the class name -class_name = 'TestStrategy' - - # This class is a sample. Feel free to customize it. class TestStrategy(IStrategy): """ @@ -45,7 +41,7 @@ class TestStrategy(IStrategy): stoploss = -0.10 # Optimal ticker interval for the strategy - ticker_interval = 5 + ticker_interval = '5m' def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """