Merge branch 'develop' into relative_stake

docs/backtesting.md

@@ -78,13 +78,17 @@ Please also read about the [strategy startup period](strategy-customization.md#s
 #### Supplying custom fee value
 
 Sometimes your account has certain fee rebates (fee reductions starting with a certain account size or monthly volume), which are not visible to ccxt.
-To account for this in backtesting, you can use `--fee 0.001` to supply this value to backtesting.
-This fee must be a percentage, and will be applied twice (once for trade entry, and once for trade exit).
+To account for this in backtesting, you can use the `--fee` command line option to supply this value to backtesting.
+This fee must be a ratio, and will be applied twice (once for trade entry, and once for trade exit).
+
+For example, if the buying and selling commission fee is 0.1% (i.e., 0.001 written as ratio), then you would run backtesting as the following:
 
 ```bash
 freqtrade backtesting --fee 0.001
 ```
 
+!!! Note
+    Only supply this option (or the corresponding configuration parameter) if you want to experiment with different fee values. By default, Backtesting fetches the default fee from the exchange pair/market info.
 
 #### Running backtest with smaller testset by using timerange
 

docs/installation.md

@@ -270,3 +270,18 @@ The easiest way is to download install Microsoft Visual Studio Community [here](
 
 Now you have an environment ready, the next step is
 [Bot Configuration](configuration.md).
+
+## Troubleshooting
+
+### MacOS installation error
+
+Newer versions of MacOS may have installation failed with errors like `error: command 'g++' failed with exit status 1`.
+
+This error will require explicit installation of the SDK Headers, which are not installed by default in this version of MacOS.
+For MacOS 10.14, this can be accomplished with the below command.
+
+``` bash
+open /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg
+```
+
+If this file is inexistant, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details.

freqtrade/configuration/config_validation.py

@@ -1,4 +1,5 @@
 import logging
+from copy import deepcopy
 from typing import Any, Dict
 
 from jsonschema import Draft4Validator, validators
@@ -42,15 +43,25 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]:
     :param conf: Config in JSON format
     :return: Returns the config if valid, otherwise throw an exception
     """
+    conf_schema = deepcopy(constants.CONF_SCHEMA)
+    if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE):
+        conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED
+    else:
+        conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED
+        # Dynamically allow empty stake-currency
+        # Since the minimal config specifies this too.
+        # It's not allowed for Dry-run or live modes
+        conf_schema['properties']['stake_currency']['enum'] += ['']  # type: ignore
+
     try:
-        FreqtradeValidator(constants.CONF_SCHEMA).validate(conf)
+        FreqtradeValidator(conf_schema).validate(conf)
         return conf
     except ValidationError as e:
         logger.critical(
             f"Invalid configuration. See config.json.example. Reason: {e}"
         )
         raise ValidationError(
-            best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message
+            best_match(Draft4Validator(conf_schema).iter_errors(conf)).message
         )
 
 

freqtrade/constants.py

@@ -275,16 +275,22 @@ CONF_SCHEMA = {
             'required': ['process_throttle_secs', 'allowed_risk']
         }
     },
-    'required': [
-        'exchange',
-        'max_open_trades',
-        'stake_currency',
-        'stake_amount',
-        'dry_run',
-        'dry_run_wallet',
-        'bid_strategy',
-        'unfilledtimeout',
-        'stoploss',
-        'minimal_roi',
-    ]
 }
+
+SCHEMA_TRADE_REQUIRED = [
+    'exchange',
+    'max_open_trades',
+    'stake_currency',
+    'stake_amount',
+    'dry_run',
+    'dry_run_wallet',
+    'bid_strategy',
+    'unfilledtimeout',
+    'stoploss',
+    'minimal_roi',
+]
+
+SCHEMA_MINIMAL_REQUIRED = [
+    'exchange',
+    'dry_run',
+]

freqtrade/data/btanalysis.py

@@ -47,7 +47,7 @@ def load_backtest_data(filename) -> pd.DataFrame:
                                       utc=True,
                                       infer_datetime_format=True
                                       )
-    df['profitabs'] = df['close_rate'] - df['open_rate']
+    df['profit'] = df['close_rate'] - df['open_rate']
     df = df.sort_values("open_time").reset_index(drop=True)
     return df
 

freqtrade/freqtradebot.py

@@ -419,8 +419,6 @@ class FreqtradeBot:
         :param pair: pair for which we want to create a LIMIT_BUY
         :return: None
         """
-        stake_currency = self.config['stake_currency']
-        fiat_currency = self.config.get('fiat_display_currency', None)
         time_in_force = self.strategy.order_time_in_force['buy']
 
         if price:
@@ -477,17 +475,6 @@ class FreqtradeBot:
             amount = order['amount']
             buy_limit_filled_price = order['price']
 
-        self.rpc.send_msg({
-            'type': RPCMessageType.BUY_NOTIFICATION,
-            'exchange': self.exchange.name.capitalize(),
-            'pair': pair,
-            'limit': buy_limit_filled_price,
-            'order_type': order_type,
-            'stake_amount': stake_amount,
-            'stake_currency': stake_currency,
-            'fiat_currency': fiat_currency
-        })
-
         # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL
         fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker')
         trade = Trade(
@@ -505,6 +492,8 @@ class FreqtradeBot:
             ticker_interval=timeframe_to_minutes(self.config['ticker_interval'])
         )
 
+        self._notify_buy(trade, order_type)
+
         # Update fees if order is closed
         if order_status == 'closed':
             self.update_trade_state(trade, order)
@@ -517,6 +506,24 @@ class FreqtradeBot:
 
         return True
 
+    def _notify_buy(self, trade: Trade, order_type: str):
+        """
+        Sends rpc notification when a buy occured.
+        """
+        msg = {
+            'type': RPCMessageType.BUY_NOTIFICATION,
+            'exchange': self.exchange.name.capitalize(),
+            'pair': trade.pair,
+            'limit': trade.open_rate,
+            'order_type': order_type,
+            'stake_amount': trade.stake_amount,
+            'stake_currency': self.config['stake_currency'],
+            'fiat_currency': self.config.get('fiat_display_currency', None),
+        }
+
+        # Send the message
+        self.rpc.send_msg(msg)
+
 #
 # SELL / exit positions / close trades logic and methods
 #
@@ -919,16 +926,16 @@ class FreqtradeBot:
             except InvalidOrderException:
                 logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}")
 
-        ordertype = self.strategy.order_types[sell_type]
+        order_type = self.strategy.order_types[sell_type]
         if sell_reason == SellType.EMERGENCY_SELL:
             # Emergencysells (default to market!)
-            ordertype = self.strategy.order_types.get("emergencysell", "market")
+            order_type = self.strategy.order_types.get("emergencysell", "market")
 
         amount = self._safe_sell_amount(trade.pair, trade.amount)
 
         # Execute sell and update trade record
         order = self.exchange.sell(pair=str(trade.pair),
-                                   ordertype=ordertype,
+                                   ordertype=order_type,
                                    amount=amount, rate=limit,
                                    time_in_force=self.strategy.order_time_in_force['sell']
                                    )
@@ -944,7 +951,7 @@ class FreqtradeBot:
         # Lock pair for one candle to prevent immediate rebuys
         self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['ticker_interval']))
 
-        self._notify_sell(trade, ordertype)
+        self._notify_sell(trade, order_type)
 
     def _notify_sell(self, trade: Trade, order_type: str):
         """
@@ -971,16 +978,13 @@ class FreqtradeBot:
             'profit_percent': profit_percent,
             'sell_reason': trade.sell_reason,
             'open_date': trade.open_date,
-            'close_date': trade.close_date or datetime.utcnow()
+            'close_date': trade.close_date or datetime.utcnow(),
+            'stake_currency': self.config['stake_currency'],
         }
 
-        # For regular case, when the configuration exists
-        if 'stake_currency' in self.config and 'fiat_display_currency' in self.config:
-            stake_currency = self.config['stake_currency']
-            fiat_currency = self.config['fiat_display_currency']
+        if 'fiat_display_currency' in self.config:
             msg.update({
-                'stake_currency': stake_currency,
-                'fiat_currency': fiat_currency,
+                'fiat_currency': self.config['fiat_display_currency'],
             })
 
         # Send the message

freqtrade/plot/plotting.py

@@ -120,8 +120,8 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
             )
         )
         # Create description for sell summarizing the trade
-        desc = trades.apply(lambda row: f"{round(row['profitperc'], 3)}%, {row['sell_reason']}, "
-                                        f"{row['duration']} min",
+        desc = trades.apply(lambda row: f"{round(row['profitperc'] * 100, 1)}%, "
+                                        f"{row['sell_reason']}, {row['duration']} min",
                             axis=1)
         trade_sells = go.Scatter(
             x=trades["close_time"],

freqtrade/utils.py

@@ -12,7 +12,8 @@ from colorama import init as colorama_init
 from tabulate import tabulate
 
 from freqtrade.configuration import (Configuration, TimeRange,
-                                     remove_credentials)
+                                     remove_credentials,
+                                     validate_config_consistency)
 from freqtrade.configuration.directory_operations import (copy_sample_files,
                                                           create_userdata_dir)
 from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY
@@ -40,6 +41,7 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
 
     # Ensure we do not use Exchange credentials
     remove_credentials(config)
+    validate_config_consistency(config)
 
     return config
 

tests/data/test_btanalysis.py

@@ -20,7 +20,7 @@ def test_load_backtest_data(testdatadir):
     filename = testdatadir / "backtest-result_test.json"
     bt_data = load_backtest_data(filename)
     assert isinstance(bt_data, DataFrame)
-    assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"]
+    assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profit"]
     assert len(bt_data) == 179
 
     # Test loading from string (must yield same result)

tests/test_configuration.py

@@ -49,6 +49,7 @@ def test_load_config_missing_attributes(default_conf) -> None:
 
     conf = deepcopy(default_conf)
     conf.pop('stake_currency')
+    conf['runmode'] = RunMode.DRY_RUN
     with pytest.raises(ValidationError, match=r".*'stake_currency' is a required property.*"):
         validate_config_schema(conf)
 

tests/test_plotting.py

@@ -119,6 +119,7 @@ def test_plot_trades(testdatadir, caplog):
     assert trade_sell.yaxis == 'y'
     assert len(trades) == len(trade_sell.x)
     assert trade_sell.marker.color == 'red'
+    assert trade_sell.text[0] == "4.0%, roi, 15 min"
 
 
 def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, testdatadir, caplog):