diff --git a/config_full.json.example b/config_full.json.example index 9dba8f539..b0719bcc6 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -33,6 +33,11 @@ "order_book_min": 1, "order_book_max": 9 }, + "order_types": { + "buy": "limit", + "sell": "limit", + "stoploss": "market" + }, "exchange": { "name": "bittrex", "key": "your_exchange_key", diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 31fea17fc..ec8873b12 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -204,6 +204,8 @@ optional arguments: number) --timerange TIMERANGE specify what timerange of data to use. + --hyperopt PATH specify hyperopt file (default: + freqtrade/optimize/default_hyperopt.py) -e INT, --epochs INT specify number of epochs (default: 100) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate diff --git a/docs/configuration.md b/docs/configuration.md index f40c2c338..62559a41e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,6 +39,7 @@ The table below will list all configuration parameters. | `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. +| `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`). | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. @@ -138,6 +139,22 @@ use the `last` price and values between those interpolate between ask and last price. Using `ask` price will guarantee quick success in bid, but bot will also end up paying more then would probably have been necessary. +### Understand order_types + +`order_types` contains a dict mapping order-types to market-types. This allows to buy using limit orders, sell using limit-orders, and create stoploss orders using market. +This can be set in the configuration or in the strategy. Configuration overwrites strategy configurations. + +If this is configured, all 3 values (`"buy"`, `"sell"` and `"stoploss"`) need to be present, otherwise the bot warn about it and will fail to start. +The below is the default which is used if this is not configured in either Strategy or configuration. + +``` json + "order_types": { + "buy": "limit", + "sell": "limit", + "stoploss": "market" + }, +``` + ### What values for exchange.name? Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports 115 cryptocurrency diff --git a/docs/hyperopt.md b/docs/hyperopt.md index e2dcf3e95..dffe84d1d 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -19,18 +19,27 @@ and still take a long time. ## Prepare Hyperopting -We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) +Before we start digging in Hyperopt, we recommend you to take a look at +an example hyperopt file located into [user_data/hyperopts/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) + +### 1. Install a Custom Hyperopt File +This is very simple. Put your hyperopt file into the folder +`user_data/hyperopts`. + +Let assume you want a hyperopt file `awesome_hyperopt.py`: +1. Copy the file `user_data/hyperopts/sample_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` + -### Configure your Guards and Triggers +### 2. Configure your Guards and Triggers +There are two places you need to change in your hyperopt file to add a +new buy hyperopt for testing: +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). +- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). -There are two places you need to change to add a new buy strategy for testing: -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L231-L264). -- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L213-L224) -and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. +There you have two different types of indicators: 1. `guards` and 2. `triggers`. -There you have two different type of indicators: 1. `guards` and 2. `triggers`. -1. Guards are conditions like "never buy if ADX < 10", or "never buy if -current price is over EMA10". +1. Guards are conditions like "never buy if ADX < 10", or never buy if +current price is over EMA10. 2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower bollinger band". @@ -124,9 +133,12 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -python3 ./freqtrade/main.py -c config.json hyperopt -e 5000 +python3 ./freqtrade/main.py -s --hyperopt -c config.json hyperopt -e 5000 ``` +Use `` and `` as the names of the custom strategy +(only required for generating sells) and the custom hyperopt used. + The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 8e26752fe..84e1a0f77 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -104,6 +104,14 @@ class Arguments(object): type=str, metavar='PATH', ) + self.parser.add_argument( + '--customhyperopt', + help='specify hyperopt class name (default: %(default)s)', + dest='hyperopt', + default=constants.DEFAULT_HYPEROPT, + type=str, + metavar='NAME', + ) self.parser.add_argument( '--dynamic-whitelist', help='dynamically generate and update whitelist' diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 0a7bb7f80..feec0cb43 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -53,6 +53,9 @@ class Configuration(object): if self.args.strategy_path: config.update({'strategy_path': self.args.strategy_path}) + # Add the hyperopt file to use + config.update({'hyperopt': self.args.hyperopt}) + # Load Common configuration config = self._load_common_config(config) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 2c753190e..055fee3b2 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,9 +9,12 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' +DEFAULT_HYPEROPT = 'DefaultHyperOpts' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' +REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss'] +ORDERTYPE_POSSIBILITIES = ['limit', 'market'] TICKER_INTERVAL_MINUTES = { @@ -101,6 +104,15 @@ CONF_SCHEMA = { 'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50} } }, + 'order_types': { + 'type': 'object', + 'properties': { + 'buy': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, + 'sell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, + 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES} + }, + 'required': ['buy', 'sell', 'stoploss'] + }, 'exchange': {'$ref': '#/definitions/exchange'}, 'edge': {'$ref': '#/definitions/edge'}, 'experimental': { diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index dedaa19a3..009b80664 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -1,8 +1,7 @@ # pragma pylint: disable=W0603 """ Edge positioning package """ import logging -from typing import Any, Dict -from collections import namedtuple +from typing import Any, Dict, NamedTuple import arrow import numpy as np @@ -18,6 +17,16 @@ from freqtrade.strategy.interface import SellType logger = logging.getLogger(__name__) +class PairInfo(NamedTuple): + stoploss: float + winrate: float + risk_reward_ratio: float + required_risk_reward: float + expectancy: float + nb_trades: int + avg_trade_duration: float + + class Edge(): """ Calculates Win Rate, Risk Reward Ratio, Expectancy @@ -30,13 +39,6 @@ class Edge(): config: Dict = {} _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - # pair info data type - _pair_info = namedtuple( - 'pair_info', - ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy', - 'nb_trades', 'avg_trade_duration'] - ) - def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: self.config = config @@ -294,16 +296,15 @@ class Edge(): final = {} for x in df.itertuples(): - info = { - 'stoploss': x.stoploss, - 'winrate': x.winrate, - 'risk_reward_ratio': x.risk_reward_ratio, - 'required_risk_reward': x.required_risk_reward, - 'expectancy': x.expectancy, - 'nb_trades': x.nb_trades, - 'avg_trade_duration': x.avg_trade_duration - } - final[x.pair] = self._pair_info(**info) + final[x.pair] = PairInfo( + x.stoploss, + x.winrate, + x.risk_reward_ratio, + x.required_risk_reward, + x.expectancy, + x.nb_trades, + x.avg_trade_duration + ) # Returning a list of pairs in order of "expectancy" return final diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 4af9db6db..ae07e36e9 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -102,7 +102,7 @@ class Exchange(object): self.markets = self._load_markets() # Check if all pairs are available self.validate_pairs(config['exchange']['pair_whitelist']) - + self.validate_ordertypes(config.get('order_types', {})) if config.get('ticker_interval'): # Check if timeframe is available self.validate_timeframes(config['ticker_interval']) @@ -218,6 +218,15 @@ class Exchange(object): raise OperationalException( f'Invalid ticker {timeframe}, this Exchange supports {timeframes}') + def validate_ordertypes(self, order_types: Dict) -> None: + """ + Checks if order-types configured in strategy/config are supported + """ + if any(v == 'market' for k, v in order_types.items()): + if not self.exchange_has('createMarketOrder'): + raise OperationalException( + f'Exchange {self.name} does not support market orders.') + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. @@ -249,14 +258,14 @@ class Exchange(object): price = ceil(big_price) / pow(10, symbol_prec) return price - def buy(self, pair: str, rate: float, amount: float) -> Dict: + def buy(self, pair: str, ordertype: str, amount: float, rate: float) -> Dict: if self._conf['dry_run']: order_id = f'dry_run_buy_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { 'pair': pair, 'price': rate, 'amount': amount, - 'type': 'limit', + 'type': ordertype, 'side': 'buy', 'remaining': 0.0, 'datetime': arrow.utcnow().isoformat(), @@ -268,9 +277,9 @@ class Exchange(object): try: # Set the precision for amount and price(rate) as accepted by the exchange amount = self.symbol_amount_prec(pair, amount) - rate = self.symbol_price_prec(pair, rate) + rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None - return self._api.create_limit_buy_order(pair, amount, rate) + return self._api.create_order(pair, ordertype, 'buy', amount, rate) except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create limit buy order on market {pair}.' @@ -287,14 +296,14 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) - def sell(self, pair: str, rate: float, amount: float) -> Dict: + def sell(self, pair: str, ordertype: str, amount: float, rate: float) -> Dict: if self._conf['dry_run']: order_id = f'dry_run_sell_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { 'pair': pair, 'price': rate, 'amount': amount, - 'type': 'limit', + 'type': ordertype, 'side': 'sell', 'remaining': 0.0, 'datetime': arrow.utcnow().isoformat(), @@ -305,9 +314,9 @@ class Exchange(object): try: # Set the precision for amount and price(rate) as accepted by the exchange amount = self.symbol_amount_prec(pair, amount) - rate = self.symbol_price_prec(pair, rate) + rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None - return self._api.create_limit_sell_order(pair, amount, rate) + return self._api.create_order(pair, ordertype, 'sell', amount, rate) except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create limit sell order on market {pair}.' diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index eb92375ec..8a2db84a9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,6 +17,7 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange +from freqtrade.wallets import Wallets from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType @@ -56,6 +57,7 @@ class FreqtradeBot(object): self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) + self.wallets = Wallets(self.exchange) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -194,9 +196,6 @@ class FreqtradeBot(object): self.edge.calculate() self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) - # Refreshing candles - self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) - # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -338,7 +337,9 @@ class FreqtradeBot(object): else: stake_amount = self.config['stake_amount'] + # TODO: should come from the wallet avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) + # avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) @@ -475,7 +476,8 @@ class FreqtradeBot(object): amount = stake_amount / buy_limit - order_id = self.exchange.buy(pair, buy_limit, amount)['id'] + order_id = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], + amount=amount, rate=buy_limit)['id'] self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, @@ -505,6 +507,10 @@ class FreqtradeBot(object): ) Trade.session.add(trade) Trade.session.flush() + + # Updating wallets + self.wallets.update() + return True def process_maybe_execute_buy(self) -> bool: @@ -549,7 +555,14 @@ class FreqtradeBot(object): if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair - return self.handle_trade(trade) + result = self.handle_trade(trade) + + # Updating wallets if any trade occured + if result: + self.wallets.update() + + return result + except DependencyException as exception: logger.warning('Unable to sell trade: %s', exception) return False @@ -688,14 +701,17 @@ class FreqtradeBot(object): # Check if trade is still actually open if int(order['remaining']) == 0: + self.wallets.update() continue # Check if trade is still actually open if order['status'] == 'open': if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: self.handle_timedout_limit_buy(trade, order) + self.wallets.update() elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: self.handle_timedout_limit_sell(trade, order) + self.wallets.update() # FIX: 20180110, why is cancel.order unconditionally here, whereas # it is conditionally called in the @@ -762,8 +778,13 @@ class FreqtradeBot(object): :param sellreason: Reason the sell was triggered :return: None """ + sell_type = 'sell' + if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): + sell_type = 'stoploss' # Execute sell and update trade record - order_id = self.exchange.sell(str(trade.pair), limit, trade.amount)['id'] + order_id = self.exchange.sell(pair=str(trade.pair), + ordertype=self.strategy.order_types[sell_type], + amount=trade.amount, rate=limit)['id'] trade.open_order_id = order_id trade.close_rate_requested = limit trade.sell_reason = sell_reason.value diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 52766f78e..b1407de18 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -20,6 +20,7 @@ from pandas import DataFrame from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange +from freqtrade.optimize.default_hyperopt import DefaultHyperOpts # noqa: F401 logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py new file mode 100644 index 000000000..6139f8140 --- /dev/null +++ b/freqtrade/optimize/default_hyperopt.py @@ -0,0 +1,130 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable, List +from functools import reduce + +from skopt.space import Categorical, Dimension, Integer, Real + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.optimize.hyperopt_interface import IHyperOpt + +class_name = 'DefaultHyperOpts' + + +class DefaultHyperOpts(IHyperOpt): + """ + Default hyperopt provided by freqtrade bot. + You can override it with your own hyperopt + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['adx'] = ta.ADX(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['sar'] = ta.SAR(dataframe) + return dataframe + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd-enabled' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) + + # TRIGGERS + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + ] + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> List[Dimension]: + """ + Stoploss Value to search + """ + return [ + Real(-0.5, -0.02, name='stoploss'), + ] + + @staticmethod + def roi_space() -> List[Dimension]: + """ + Values to search for each ROI steps + """ + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b2d05d603..70d20673c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -9,22 +9,21 @@ import multiprocessing import os import sys from argparse import Namespace -from functools import reduce from math import exp from operator import itemgetter -from typing import Any, Callable, Dict, List +from typing import Any, Dict, List -import talib.abstract as ta from pandas import DataFrame -from sklearn.externals.joblib import Parallel, delayed, dump, load +from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects from skopt import Optimizer -from skopt.space import Categorical, Dimension, Integer, Real +from skopt.space import Dimension -import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting +from freqtrade.optimize.hyperopt_resolver import HyperOptResolver + logger = logging.getLogger(__name__) @@ -42,6 +41,9 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) + self.config = config + self.custom_hyperopt = HyperOptResolver(self.config).hyperopt + # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days self.target_trades = 600 @@ -74,24 +76,6 @@ class Hyperopt(Backtesting): arg_dict = {dim.name: value for dim, value in zip(dimensions, params)} return arg_dict - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe['adx'] = ta.ADX(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['sar'] = ta.SAR(dataframe) - - return dataframe - def save_trials(self) -> None: """ Save hyperopt trials to file @@ -121,7 +105,8 @@ class Hyperopt(Backtesting): best_result['params'] ) if 'roi_t1' in best_result['params']: - logger.info('ROI table:\n%s', self.generate_roi_table(best_result['params'])) + logger.info('ROI table:\n%s', + self.custom_hyperopt.generate_roi_table(best_result['params'])) def log_results(self, results) -> None: """ @@ -149,59 +134,6 @@ class Hyperopt(Backtesting): result = trade_loss + profit_loss + duration_loss return result - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def roi_space() -> List[Dimension]: - """ - Values to search for each ROI steps - """ - return [ - Integer(10, 120, name='roi_t1'), - Integer(10, 60, name='roi_t2'), - Integer(10, 40, name='roi_t3'), - Real(0.01, 0.04, name='roi_p1'), - Real(0.01, 0.07, name='roi_p2'), - Real(0.01, 0.20, name='roi_p3'), - ] - - @staticmethod - def stoploss_space() -> List[Dimension]: - """ - Stoploss search space - """ - return [ - Real(-0.5, -0.02, name='stoploss'), - ] - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - def has_space(self, space: str) -> bool: """ Tell if a space value is contained in the configuration @@ -216,61 +148,20 @@ class Hyperopt(Backtesting): """ spaces: List[Dimension] = [] if self.has_space('buy'): - spaces += Hyperopt.indicator_space() + spaces += self.custom_hyperopt.indicator_space() if self.has_space('roi'): - spaces += Hyperopt.roi_space() + spaces += self.custom_hyperopt.roi_space() if self.has_space('stoploss'): - spaces += Hyperopt.stoploss_space() + spaces += self.custom_hyperopt.stoploss_space() return spaces - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use - """ - conditions = [] - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - def generate_optimizer(self, _params) -> Dict: + def generate_optimizer(self, _params: Dict) -> Dict: params = self.get_args(_params) - if self.has_space('roi'): - self.strategy.minimal_roi = self.generate_roi_table(params) + self.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) if self.has_space('buy'): - self.advise_buy = self.buy_strategy_generator(params) + self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] @@ -329,7 +220,8 @@ class Hyperopt(Backtesting): ) def run_optimizer_parallel(self, parallel, asked) -> List: - return parallel(delayed(self.generate_optimizer)(v) for v in asked) + return parallel(delayed( + wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) def load_previous_results(self): """ read trials file if we have one """ @@ -351,7 +243,8 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.strategy.advise_indicators = Hyperopt.populate_indicators # type: ignore + self.strategy.advise_indicators = \ + self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py new file mode 100644 index 000000000..d42206658 --- /dev/null +++ b/freqtrade/optimize/hyperopt_interface.py @@ -0,0 +1,66 @@ +""" +IHyperOpt interface +This module defines the interface to apply for hyperopts +""" + +from abc import ABC, abstractmethod +from typing import Dict, Any, Callable, List + +from pandas import DataFrame +from skopt.space import Dimension + + +class IHyperOpt(ABC): + """ + Interface for freqtrade hyperopts + Defines the mandatory structure must follow any custom strategies + + Attributes you can use: + minimal_roi -> Dict: Minimal ROI designed for the strategy + stoploss -> float: optimal stoploss designed for the strategy + ticker_interval -> int: value of the ticker interval to use for the strategy + """ + + @staticmethod + @abstractmethod + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Populate indicators that will be used in the Buy and Sell strategy + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :return: a Dataframe with all mandatory indicators for the strategies + """ + + @staticmethod + @abstractmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Create a buy strategy generator + """ + + @staticmethod + @abstractmethod + def indicator_space() -> List[Dimension]: + """ + Create an indicator space + """ + + @staticmethod + @abstractmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Create an roi table + """ + + @staticmethod + @abstractmethod + def stoploss_space() -> List[Dimension]: + """ + Create a stoploss space + """ + + @staticmethod + @abstractmethod + def roi_space() -> List[Dimension]: + """ + Create a roi space + """ diff --git a/freqtrade/optimize/hyperopt_resolver.py b/freqtrade/optimize/hyperopt_resolver.py new file mode 100644 index 000000000..3d019e8df --- /dev/null +++ b/freqtrade/optimize/hyperopt_resolver.py @@ -0,0 +1,104 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import importlib.util +import inspect +import logging +import os +from typing import Optional, Dict, Type + +from freqtrade.constants import DEFAULT_HYPEROPT +from freqtrade.optimize.hyperopt_interface import IHyperOpt + + +logger = logging.getLogger(__name__) + + +class HyperOptResolver(object): + """ + This class contains all the logic to load custom hyperopt class + """ + + __slots__ = ['hyperopt'] + + def __init__(self, config: Optional[Dict] = None) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + config = config or {} + + # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt + hyperopt_name = config.get('hyperopt') or DEFAULT_HYPEROPT + self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) + + def _load_hyperopt( + self, hyperopt_name: str, extra_dir: Optional[str] = None) -> IHyperOpt: + """ + Search and loads the specified hyperopt. + :param hyperopt_name: name of the module to import + :param extra_dir: additional directory to search for the given hyperopt + :return: HyperOpt instance or None + """ + current_path = os.path.dirname(os.path.realpath(__file__)) + abs_paths = [ + os.path.join(current_path, '..', '..', 'user_data', 'hyperopts'), + current_path, + ] + + if extra_dir: + # Add extra hyperopt directory on top of search paths + abs_paths.insert(0, extra_dir) + + for path in abs_paths: + hyperopt = self._search_hyperopt(path, hyperopt_name) + if hyperopt: + logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, path) + return hyperopt + + raise ImportError( + "Impossible to load Hyperopt '{}'. This class does not exist" + " or contains Python code errors".format(hyperopt_name) + ) + + @staticmethod + def _get_valid_hyperopts(module_path: str, hyperopt_name: str) -> Optional[Type[IHyperOpt]]: + """ + Returns a list of all possible hyperopts for the given module_path + :param module_path: absolute path to the module + :param hyperopt_name: Class name of the hyperopt + :return: Tuple with (name, class) or None + """ + + # Generate spec based on absolute path + spec = importlib.util.spec_from_file_location('user_data.hyperopts', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore # importlib does not use typehints + + valid_hyperopts_gen = ( + obj for name, obj in inspect.getmembers(module, inspect.isclass) + if hyperopt_name == name and IHyperOpt in obj.__bases__ + ) + return next(valid_hyperopts_gen, None) + + @staticmethod + def _search_hyperopt(directory: str, hyperopt_name: str) -> Optional[IHyperOpt]: + """ + Search for the hyperopt_name in the given directory + :param directory: relative or absolute directory path + :return: name of the hyperopt class + """ + logger.debug('Searching for hyperopt %s in \'%s\'', hyperopt_name, directory) + for entry in os.listdir(directory): + # Only consider python files + if not entry.endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + hyperopt = HyperOptResolver._get_valid_hyperopts( + os.path.abspath(os.path.join(directory, entry)), hyperopt_name + ) + if hyperopt: + return hyperopt() + return None diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index f1646779b..b282a5938 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -28,6 +28,13 @@ class DefaultStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' + # Optional order type mapping + order_types = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'limit' + } + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index d1d4703a4..9b7b180cc 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -75,6 +75,13 @@ class IStrategy(ABC): # associated ticker interval ticker_interval: str + # Optional order types + order_types: Dict = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'limit' + } + # run "populate_indicators" only for new candle process_only_new_candles: bool = False diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index aee47580c..3f25e4838 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -75,6 +75,19 @@ class StrategyResolver(object): else: config['process_only_new_candles'] = self.strategy.process_only_new_candles + if 'order_types' in config: + self.strategy.order_types = config['order_types'] + logger.info( + "Override strategy 'order_types' with value in config file: %s.", + config['order_types'] + ) + else: + config['order_types'] = self.strategy.order_types + + if not all(k in self.strategy.order_types for k in constants.REQUIRED_ORDERTYPES): + raise ImportError(f"Impossible to load Strategy '{self.strategy.__class__.__name__}'. " + f"Order-types mapping is incomplete.") + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 8a497725f..b6c022b45 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -30,6 +30,7 @@ def log_has(line, logs): def patch_exchange(mocker, api_mock=None) -> None: mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_ordertypes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value="Bittrex")) mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value="bittrex")) if api_mock: diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 14c9114c3..fac055c17 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -4,7 +4,7 @@ import pytest import logging from freqtrade.tests.conftest import get_patched_freqtradebot -from freqtrade.edge import Edge +from freqtrade.edge import Edge, PairInfo from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, @@ -128,9 +128,9 @@ def test_adjust(mocker, default_conf): edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) + 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'C/D': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'N/O': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) } )) @@ -143,9 +143,9 @@ def test_stoploss(mocker, default_conf): edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) + 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'C/D': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'N/O': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) } )) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 788ef4518..207f14efe 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -355,6 +355,36 @@ def test_validate_timeframes_not_in_config(default_conf, mocker): Exchange(default_conf) +def test_validate_order_types(default_conf, mocker): + api_mock = MagicMock() + + type(api_mock).has = PropertyMock(return_value={'createMarketOrder': True}) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + default_conf['order_types'] = {'buy': 'limit', 'sell': 'limit', 'stoploss': 'market'} + Exchange(default_conf) + + type(api_mock).has = PropertyMock(return_value={'createMarketOrder': False}) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + + default_conf['order_types'] = {'buy': 'limit', 'sell': 'limit', 'stoploss': 'market'} + + with pytest.raises(OperationalException, + match=r'Exchange .* does not support market orders.'): + Exchange(default_conf) + + +def test_validate_order_types_not_in_config(default_conf, mocker): + api_mock = MagicMock() + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + + conf = copy.deepcopy(default_conf) + Exchange(conf) + + def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') @@ -373,7 +403,7 @@ def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) - order = exchange.buy(pair='ETH/BTC', rate=200, amount=1) + order = exchange.buy(pair='ETH/BTC', ordertype='limit', amount=1, rate=200) assert 'id' in order assert 'dry_run_buy_' in order['id'] @@ -381,47 +411,64 @@ def test_buy_dry_run(default_conf, mocker): def test_buy_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - api_mock.create_limit_buy_order = MagicMock(return_value={ + order_type = 'market' + api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { 'foo': 'bar' } }) default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock) - order = exchange.buy(pair='ETH/BTC', rate=200, amount=1) + order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert 'id' in order assert 'info' in order assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'buy' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] is None + + api_mock.create_order.reset_mock() + order_type = 'limit' + order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'buy' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 # test exception handling with pytest.raises(DependencyException): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.InsufficientFunds) + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(DependencyException): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.InvalidOrder) + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(TemporaryError): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.NetworkError) + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(OperationalException): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.BaseError) + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) def test_sell_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) - order = exchange.sell(pair='ETH/BTC', rate=200, amount=1) + order = exchange.sell(pair='ETH/BTC', ordertype='limit', amount=1, rate=200) assert 'id' in order assert 'dry_run_sell_' in order['id'] @@ -429,7 +476,8 @@ def test_sell_dry_run(default_conf, mocker): def test_sell_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) - api_mock.create_limit_sell_order = MagicMock(return_value={ + order_type = 'market' + api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { 'foo': 'bar' @@ -438,32 +486,48 @@ def test_sell_prod(default_conf, mocker): default_conf['dry_run'] = False exchange = get_patched_exchange(mocker, default_conf, api_mock) + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - order = exchange.sell(pair='ETH/BTC', rate=200, amount=1) + order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert 'id' in order assert 'info' in order assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] is None + + api_mock.create_order.reset_mock() + order_type = 'limit' + order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 # test exception handling with pytest.raises(DependencyException): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.InsufficientFunds) + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(DependencyException): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.InvalidOrder) + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(TemporaryError): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.NetworkError) + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(OperationalException): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.BaseError) + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) def test_get_balance_dry_run(default_conf, mocker): diff --git a/freqtrade/tests/optimize/test_edge_cli.py b/freqtrade/tests/optimize/test_edge_cli.py index f8db3dec4..0d0f64e0c 100644 --- a/freqtrade/tests/optimize/test_edge_cli.py +++ b/freqtrade/tests/optimize/test_edge_cli.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import json from typing import List -from freqtrade.edge import Edge +from freqtrade.edge import PairInfo from freqtrade.arguments import Arguments from freqtrade.optimize.edge_cli import (EdgeCli, setup_configuration, start) from freqtrade.tests.conftest import log_has, patch_exchange @@ -123,17 +123,8 @@ def test_generate_edge_table(edge_conf, mocker): edge_cli = EdgeCli(edge_conf) results = {} - info = { - 'stoploss': -0.01, - 'winrate': 0.60, - 'risk_reward_ratio': 2, - 'required_risk_reward': 1, - 'expectancy': 3, - 'nb_trades': 10, - 'avg_trade_duration': 60 - } + results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) - results['ETH/BTC'] = Edge._pair_info(**info) assert edge_cli._generate_edge_table(results).count(':|') == 7 assert edge_cli._generate_edge_table(results).count('| ETH/BTC |') == 1 assert edge_cli._generate_edge_table(results).count( diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index c93f2d316..85d140b6d 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -175,7 +175,7 @@ def test_roi_table_generation(hyperopt) -> None: 'roi_p3': 3, } - assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} + assert hyperopt.custom_hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: @@ -243,7 +243,8 @@ def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) - dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe @@ -255,9 +256,10 @@ def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) - dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) - populate_buy_trend = hyperopt.buy_strategy_generator( + populate_buy_trend = hyperopt.custom_hyperopt.buy_strategy_generator( { 'adx-value': 20, 'fastd-value': 20, diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index abc531689..a38050f24 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -88,8 +88,8 @@ def test_load_strategy_invalid_directory(result, caplog): def test_load_not_found_strategy(): strategy = StrategyResolver() with pytest.raises(ImportError, - match=r'Impossible to load Strategy \'NotFoundStrategy\'.' - r' This class does not exist or contains Python code errors'): + match=r"Impossible to load Strategy 'NotFoundStrategy'." + r" This class does not exist or contains Python code errors"): strategy._load_strategy(strategy_name='NotFoundStrategy', config={}) @@ -182,6 +182,42 @@ def test_strategy_override_process_only_new_candles(caplog): ) in caplog.record_tuples +def test_strategy_override_order_types(caplog): + caplog.set_level(logging.INFO) + + order_types = { + 'buy': 'market', + 'sell': 'limit', + 'stoploss': 'limit' + } + + config = { + 'strategy': 'DefaultStrategy', + 'order_types': order_types + } + resolver = StrategyResolver(config) + + assert resolver.strategy.order_types + for method in ['buy', 'sell', 'stoploss']: + assert resolver.strategy.order_types[method] == order_types[method] + + assert ('freqtrade.strategy.resolver', + logging.INFO, + "Override strategy 'order_types' with value in config file:" + " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'}." + ) in caplog.record_tuples + + config = { + 'strategy': 'DefaultStrategy', + 'order_types': {'buy': 'market'} + } + # Raise error for invalid configuration + with pytest.raises(ImportError, + match=r"Impossible to load Strategy 'DefaultStrategy'. " + r"Order-types mapping is incomplete."): + StrategyResolver(config) + + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 4cba9e308..cef89c250 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -553,7 +553,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade) freqtrade.create_trade() - rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] + rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount'] assert rate * amount >= default_conf['stake_amount'] @@ -863,10 +863,10 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert freqtrade.execute_buy(pair, stake_amount) assert get_bid.call_count == 1 assert buy_mm.call_count == 1 - call_args = buy_mm.call_args_list[0][0] - assert call_args[0] == pair - assert call_args[1] == bid - assert call_args[2] == stake_amount / bid + call_args = buy_mm.call_args_list[0][1] + assert call_args['pair'] == pair + assert call_args['rate'] == bid + assert call_args['amount'] == stake_amount / bid # Test calling with price fix_price = 0.06 @@ -875,10 +875,10 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert get_bid.call_count == 1 assert buy_mm.call_count == 2 - call_args = buy_mm.call_args_list[1][0] - assert call_args[0] == pair - assert call_args[1] == fix_price - assert call_args[2] == stake_amount / fix_price + call_args = buy_mm.call_args_list[1][1] + assert call_args['pair'] == pair + assert call_args['rate'] == fix_price + assert call_args['amount'] == stake_amount / fix_price def test_process_maybe_execute_buy(mocker, default_conf) -> None: diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py new file mode 100644 index 000000000..cc10d665c --- /dev/null +++ b/freqtrade/tests/test_wallets.py @@ -0,0 +1,84 @@ +# pragma pylint: disable=missing-docstring +from freqtrade.tests.conftest import get_patched_freqtradebot +from unittest.mock import MagicMock + + +def test_sync_wallet_at_boot(mocker, default_conf): + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value={ + "BNT": { + "free": 1.0, + "used": 2.0, + "total": 3.0 + }, + "GAS": { + "free": 0.260739, + "used": 0.0, + "total": 0.260739 + }, + }) + ) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + assert len(freqtrade.wallets.wallets) == 2 + assert freqtrade.wallets.wallets['BNT'].free == 1.0 + assert freqtrade.wallets.wallets['BNT'].used == 2.0 + assert freqtrade.wallets.wallets['BNT'].total == 3.0 + assert freqtrade.wallets.wallets['GAS'].free == 0.260739 + assert freqtrade.wallets.wallets['GAS'].used == 0.0 + assert freqtrade.wallets.wallets['GAS'].total == 0.260739 + + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value={ + "BNT": { + "free": 1.2, + "used": 1.9, + "total": 3.5 + }, + "GAS": { + "free": 0.270739, + "used": 0.1, + "total": 0.260439 + }, + }) + ) + + freqtrade.wallets.update() + + assert len(freqtrade.wallets.wallets) == 2 + assert freqtrade.wallets.wallets['BNT'].free == 1.2 + assert freqtrade.wallets.wallets['BNT'].used == 1.9 + assert freqtrade.wallets.wallets['BNT'].total == 3.5 + assert freqtrade.wallets.wallets['GAS'].free == 0.270739 + assert freqtrade.wallets.wallets['GAS'].used == 0.1 + assert freqtrade.wallets.wallets['GAS'].total == 0.260439 + + +def test_sync_wallet_missing_data(mocker, default_conf): + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value={ + "BNT": { + "free": 1.0, + "used": 2.0, + "total": 3.0 + }, + "GAS": { + "free": 0.260739, + "total": 0.260739 + }, + }) + ) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + assert len(freqtrade.wallets.wallets) == 2 + assert freqtrade.wallets.wallets['BNT'].free == 1.0 + assert freqtrade.wallets.wallets['BNT'].used == 2.0 + assert freqtrade.wallets.wallets['BNT'].total == 3.0 + assert freqtrade.wallets.wallets['GAS'].free == 0.260739 + assert freqtrade.wallets.wallets['GAS'].used is None + assert freqtrade.wallets.wallets['GAS'].total == 0.260739 diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py new file mode 100644 index 000000000..82f527d2c --- /dev/null +++ b/freqtrade/wallets.py @@ -0,0 +1,44 @@ +# pragma pylint: disable=W0603 +""" Wallet """ +import logging +from typing import Dict, Any, NamedTuple +from collections import namedtuple +from freqtrade.exchange import Exchange + +logger = logging.getLogger(__name__) + + +class Wallet(NamedTuple): + exchange: str + currency: str + free: float = 0 + used: float = 0 + total: float = 0 + + +class Wallets(object): + + # wallet data structure + wallet = namedtuple( + 'wallet', + ['exchange', 'currency', 'free', 'used', 'total'] + ) + + def __init__(self, exchange: Exchange) -> None: + self.exchange = exchange + self.wallets: Dict[str, Any] = {} + self.update() + + def update(self) -> None: + balances = self.exchange.get_balances() + + for currency in balances: + self.wallets[currency] = Wallet( + self.exchange.id, + currency, + balances[currency].get('free', None), + balances[currency].get('used', None), + balances[currency].get('total', None) + ) + + logger.info('Wallets synced ...') diff --git a/requirements.txt b/requirements.txt index 66593c264..1b271e09e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.498 +ccxt==1.17.522 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 @@ -8,6 +8,7 @@ urllib3==1.24.1 wrapt==1.10.11 pandas==0.23.4 scikit-learn==0.20.0 +joblib==0.13.0 scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 diff --git a/setup.py b/setup.py index c5f61c34d..b9e3620df 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ setup(name='freqtrade', 'pandas', 'scikit-learn', 'scipy', + 'joblib', 'jsonschema', 'TA-Lib', 'tabulate', diff --git a/user_data/hyperopts/__init__.py b/user_data/hyperopts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py new file mode 100644 index 000000000..f11236a82 --- /dev/null +++ b/user_data/hyperopts/sample_hyperopt.py @@ -0,0 +1,139 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable, List +from functools import reduce + +import numpy +from skopt.space import Categorical, Dimension, Integer, Real + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.optimize.hyperopt_interface import IHyperOpt + +class_name = 'SampleHyperOpts' + + +# This class is a sample. Feel free to customize it. +class SampleHyperOpts(IHyperOpt): + """ + This is a test hyperopt to inspire you. + More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md + You can: + - Rename the class name (Do not forget to update class_name) + - Add any methods you want to build your hyperopt + - Add any lib you need to build your hyperopt + You must keep: + - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, + roi_space, generate_roi_table, stoploss_space + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['adx'] = ta.ADX(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['sar'] = ta.SAR(dataframe) + return dataframe + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd-enabled' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) + + # TRIGGERS + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + ] + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> List[Dimension]: + """ + Stoploss Value to search + """ + return [ + Real(-0.5, -0.02, name='stoploss'), + ] + + @staticmethod + def roi_space() -> List[Dimension]: + """ + Values to search for each ROI steps + """ + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 7c3892b77..fd2e9ab75 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -48,6 +48,13 @@ class TestStrategy(IStrategy): # run "populate_indicators" only for new candle ta_on_candle = False + # Optional order type mapping + order_types = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'market' + } + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame