Merge branch 'freqtrade:develop' into develop

docs/backtesting.md

@@ -305,7 +305,7 @@ A backtesting result will look like that:
 | Sharpe                      | 2.97                |
 | Calmar                      | 6.29                |
 | Profit factor               | 1.11                |
-| Expectancy                  | -0.15               |
+| Expectancy (Ratio)          | -0.15 (-0.05)       |
 | Avg. stake amount           | 0.001      BTC      |
 | Total trade volume          | 0.429      BTC      |
 |                             |                     |
@@ -409,7 +409,7 @@ It contains some useful key metrics about performance of your strategy on backte
 | Sharpe                      | 2.97                |
 | Calmar                      | 6.29                |
 | Profit factor               | 1.11                |
-| Expectancy                  | -0.15               |
+| Expectancy (Ratio)          | -0.15 (-0.05)       |
 | Avg. stake amount           | 0.001      BTC      |
 | Total trade volume          | 0.429      BTC      |
 |                             |                     |

docs/exchanges.md

@@ -259,10 +259,17 @@ The configuration parameter `exchange.unknown_fee_rate` can be used to specify t
 
 Futures trading on bybit is currently supported for USDT markets, and will use isolated futures mode.
 Users with unified accounts (there's no way back) can create a Sub-account which will start as "non-unified", and can therefore use isolated futures.
-On startup, freqtrade will set the position mode to "One-way Mode" for the whole (sub)account. This avoids making this call over and over again (slowing down bot operations), but means that changes to this setting may result in exceptions and errors.
+On startup, freqtrade will set the position mode to "One-way Mode" for the whole (sub)account. This avoids making this call over and over again (slowing down bot operations), but means that changes to this setting may result in exceptions and errors
 
 As bybit doesn't provide funding rate history, the dry-run calculation is used for live trades as well.
 
+API Keys for live futures trading (Subaccount on non-unified) must have the following permissions:
+* Read-write
+* Contract - Orders
+* Contract - Positions
+
+We do strongly recommend to limit all API keys to the IP you're going to use it from.
+
 !!! Tip "Stoploss on Exchange"
     Bybit (futures only) supports `stoploss_on_exchange` and uses `stop-loss-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
     On futures, Bybit supports both `stop-limit` as well as `stop-market` orders. You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use.

docs/webhook-config.md

@@ -80,12 +80,18 @@ When using the Form-Encoded or JSON-Encoded configuration you can configure any
 
 The result would be a POST request with e.g. `Status: running` body and `Content-Type: text/plain` header.
 
-Optional parameters are available to enable automatic retries for webhook messages. The `webhook.retries` parameter can be set for the maximum number of retries the webhook request should attempt if it is unsuccessful (i.e. HTTP response status is not 200). By default this is set to `0` which is disabled. An additional `webhook.retry_delay` parameter can be set to specify the time in seconds between retry attempts. By default this is set to `0.1` (i.e. 100ms). Note that increasing the number of retries or retry delay may slow down the trader if there are connectivity issues with the webhook. Example configuration for retries:
+## Additional configurations
+
+The `webhook.retries` parameter can be set for the maximum number of retries the webhook request should attempt if it is unsuccessful (i.e. HTTP response status is not 200). By default this is set to `0` which is disabled. An additional `webhook.retry_delay` parameter can be set to specify the time in seconds between retry attempts. By default this is set to `0.1` (i.e. 100ms). Note that increasing the number of retries or retry delay may slow down the trader if there are connectivity issues with the webhook.
+You can also specify `webhook.timeout` - which defines how long the bot will wait until it assumes the other host as unresponsive (defaults to 10s).
+
+Example configuration for retries:
 
 ```json
   "webhook": {
         "enabled": true,
         "url": "https://<YOURHOOKURL>",
+        "timeout": 10,
         "retries": 3,
         "retry_delay": 0.2,
         "status": {
@@ -109,6 +115,8 @@ Custom messages can be sent to Webhook endpoints via the `self.dp.send_msg()` fu
 
 Different payloads can be configured for different events. Not all fields are necessary, but you should configure at least one of the dicts, otherwise the webhook will never be called.
 
+## Webhook Message types
+
 ### Entry
 
 The fields in `webhook.entry` are filled when the bot executes a long/short. Parameters are filled using string.format.

freqtrade/commands/build_config_commands.py

@@ -5,6 +5,7 @@ from typing import Any, Dict, List
 
 from questionary import Separator, prompt
 
+from freqtrade.configuration.detect_environment import running_in_docker
 from freqtrade.configuration.directory_operations import chown_user_directory
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
 from freqtrade.exceptions import OperationalException
@@ -179,7 +180,7 @@ def ask_user_config() -> Dict[str, Any]:
             "name": "api_server_listen_addr",
             "message": ("Insert Api server Listen Address (0.0.0.0 for docker, "
                         "otherwise best left untouched)"),
-            "default": "127.0.0.1",
+            "default": "127.0.0.1" if not running_in_docker() else "0.0.0.0",
             "when": lambda x: x['api_server']
         },
         {

freqtrade/configuration/detect_environment.py

@@ -0,0 +1,8 @@
+import os
+
+
+def running_in_docker() -> bool:
+    """
+    Check if we are running in a docker container
+    """
+    return os.environ.get('FT_APP_ENV') == 'docker'

freqtrade/configuration/directory_operations.py

@@ -3,6 +3,7 @@ import shutil
 from pathlib import Path
 from typing import Optional
 
+from freqtrade.configuration.detect_environment import running_in_docker
 from freqtrade.constants import (USER_DATA_FILES, USERPATH_FREQAIMODELS, USERPATH_HYPEROPTS,
                                  USERPATH_NOTEBOOKS, USERPATH_STRATEGIES, Config)
 from freqtrade.exceptions import OperationalException
@@ -30,8 +31,7 @@ def chown_user_directory(directory: Path) -> None:
     Use Sudo to change permissions of the home-directory if necessary
     Only applies when running in docker!
     """
-    import os
+    if running_in_docker():
-    if os.environ.get('FT_APP_ENV') == 'docker':
         try:
             import subprocess
             subprocess.check_output(

freqtrade/data/metrics.py

@@ -194,32 +194,35 @@ def calculate_cagr(days_passed: int, starting_balance: float, final_balance: flo
     return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1
 
 
-def calculate_expectancy(trades: pd.DataFrame) -> float:
+def calculate_expectancy(trades: pd.DataFrame) -> Tuple[float, float]:
     """
     Calculate expectancy
     :param trades: DataFrame containing trades (requires columns close_date and profit_abs)
-    :return: expectancy
+    :return: expectancy, expectancy_ratio
     """
-    if len(trades) == 0:
-        return 0
 
-    expectancy = 1
-
-    profit_sum = trades.loc[trades['profit_abs'] > 0, 'profit_abs'].sum()
-    loss_sum = abs(trades.loc[trades['profit_abs'] < 0, 'profit_abs'].sum())
-    nb_win_trades = len(trades.loc[trades['profit_abs'] > 0])
-    nb_loss_trades = len(trades.loc[trades['profit_abs'] < 0])
-
-    if (nb_win_trades > 0) and (nb_loss_trades > 0):
-        average_win = profit_sum / nb_win_trades
-        average_loss = loss_sum / nb_loss_trades
-        risk_reward_ratio = average_win / average_loss
-        winrate = nb_win_trades / len(trades)
-        expectancy = ((1 + risk_reward_ratio) * winrate) - 1
-    elif nb_win_trades == 0:
     expectancy = 0
+    expectancy_ratio = 100
 
-    return expectancy
+    if len(trades) > 0:
+        winning_trades = trades.loc[trades['profit_abs'] > 0]
+        losing_trades = trades.loc[trades['profit_abs'] < 0]
+        profit_sum = winning_trades['profit_abs'].sum()
+        loss_sum = abs(losing_trades['profit_abs'].sum())
+        nb_win_trades = len(winning_trades)
+        nb_loss_trades = len(losing_trades)
+
+        average_win = (profit_sum / nb_win_trades) if nb_win_trades > 0 else 0
+        average_loss = (loss_sum / nb_loss_trades) if nb_loss_trades > 0 else 0
+        winrate = (nb_win_trades / len(trades))
+        loserate = (nb_loss_trades / len(trades))
+
+        expectancy = (winrate * average_win) - (loserate * average_loss)
+        if (average_loss > 0):
+            risk_reward_ratio = average_win / average_loss
+            expectancy_ratio = ((1 + risk_reward_ratio) * winrate) - 1
+
+    return expectancy, expectancy_ratio
 
 
 def calculate_sortino(trades: pd.DataFrame, min_date: datetime, max_date: datetime,

freqtrade/freqai/prediction_models/LightGBMClassifier.py

@@ -32,8 +32,8 @@ class LightGBMClassifier(BaseClassifierModel):
             eval_set = None
             test_weights = None
         else:
-            eval_set = (data_dictionary["test_features"].to_numpy(),
+            eval_set = [(data_dictionary["test_features"].to_numpy(),
-                        data_dictionary["test_labels"].to_numpy()[:, 0])
+                        data_dictionary["test_labels"].to_numpy()[:, 0])]
             test_weights = data_dictionary["test_weights"]
         X = data_dictionary["train_features"].to_numpy()
         y = data_dictionary["train_labels"].to_numpy()[:, 0]
@@ -42,7 +42,6 @@ class LightGBMClassifier(BaseClassifierModel):
         init_model = self.get_init_model(dk.pair)
 
         model = LGBMClassifier(**self.model_training_parameters)
-
         model.fit(X=X, y=y, eval_set=eval_set, sample_weight=train_weights,
                   eval_sample_weight=[test_weights], init_model=init_model)
 

freqtrade/freqai/prediction_models/LightGBMRegressor.py

@@ -32,7 +32,7 @@ class LightGBMRegressor(BaseRegressionModel):
             eval_set = None
             eval_weights = None
         else:
-            eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"])
+            eval_set = [(data_dictionary["test_features"], data_dictionary["test_labels"])]
             eval_weights = data_dictionary["test_weights"]
         X = data_dictionary["train_features"]
         y = data_dictionary["train_labels"]

freqtrade/freqai/prediction_models/LightGBMRegressorMultiTarget.py

@@ -42,10 +42,10 @@ class LightGBMRegressorMultiTarget(BaseRegressionModel):
             eval_weights = [data_dictionary["test_weights"]]
             eval_sets = [(None, None)] * data_dictionary['test_labels'].shape[1]  # type: ignore
             for i in range(data_dictionary['test_labels'].shape[1]):
-                eval_sets[i] = (  # type: ignore
+                eval_sets[i] = [(  # type: ignore
                     data_dictionary["test_features"],
                     data_dictionary["test_labels"].iloc[:, i]
-                )
+                )]
 
         init_model = self.get_init_model(dk.pair)
         if init_model:

freqtrade/optimize/optimize_reports/bt_output.py

@@ -233,8 +233,9 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Calmar', f"{strat_results['calmar']:.2f}" if 'calmar' in strat_results else 'N/A'),
             ('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor'
                               in strat_results else 'N/A'),
-            ('Expectancy', f"{strat_results['expectancy']:.2f}" if 'expectancy'
+            ('Expectancy (Ratio)', (
-                           in strat_results else 'N/A'),
+                f"{strat_results['expectancy']:.2f} ({strat_results['expectancy_ratio']:.2f})" if
+                'expectancy_ratio' in strat_results else 'N/A')),
             ('Trades per day', strat_results['trades_per_day']),
             ('Avg. daily profit %',
              f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),

freqtrade/optimize/optimize_reports/optimize_reports.py

@@ -389,6 +389,7 @@ def generate_strategy_stats(pairlist: List[str],
     losing_profit = results.loc[results['profit_abs'] < 0, 'profit_abs'].sum()
     profit_factor = winning_profit / abs(losing_profit) if losing_profit else 0.0
 
+    expectancy, expectancy_ratio = calculate_expectancy(results)
     backtest_days = (max_date - min_date).days or 1
     strat_stats = {
         'trades': results.to_dict(orient='records'),
@@ -414,7 +415,8 @@ def generate_strategy_stats(pairlist: List[str],
         'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(),
         'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(),
         'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']),
-        'expectancy': calculate_expectancy(results),
+        'expectancy': expectancy,
+        'expectancy_ratio': expectancy_ratio,
         'sortino': calculate_sortino(results, min_date, max_date, start_balance),
         'sharpe': calculate_sharpe(results, min_date, max_date, start_balance),
         'calmar': calculate_calmar(results, min_date, max_date, start_balance),

freqtrade/rpc/rpc.py

@@ -18,7 +18,7 @@ from freqtrade import __version__
 from freqtrade.configuration.timerange import TimeRange
 from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT, Config
 from freqtrade.data.history import load_data
-from freqtrade.data.metrics import calculate_max_drawdown
+from freqtrade.data.metrics import calculate_expectancy, calculate_max_drawdown
 from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirection, SignalDirection,
                              State, TradingMode)
 from freqtrade.exceptions import ExchangeError, PricingError
@@ -523,20 +523,14 @@ class RPC:
 
         profit_factor = winning_profit / abs(losing_profit) if losing_profit else float('inf')
 
-        mean_winning_profit = (winning_profit / winning_trades) if winning_trades > 0 else 0
-        mean_losing_profit = (abs(losing_profit) / losing_trades) if losing_trades > 0 else 0
-
         winrate = (winning_trades / closed_trade_count) if closed_trade_count > 0 else 0
-        loserate = (1 - winrate)
-
-        expectancy, expectancy_ratio = self.__calc_expectancy(mean_winning_profit,
-                                                              mean_losing_profit,
-                                                              winrate,
-                                                              loserate)
 
         trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT),
                                 'profit_abs': trade.close_profit_abs}
                                for trade in trades if not trade.is_open and trade.close_date])
+
+        expectancy, expectancy_ratio = calculate_expectancy(trades_df)
+
         max_drawdown_abs = 0.0
         max_drawdown = 0.0
         if len(trades_df) > 0:
@@ -625,23 +619,6 @@ class RPC:
 
         return est_stake, est_bot_stake
 
-    def __calc_expectancy(
-            self, mean_winning_profit: float, mean_losing_profit: float,
-            winrate: float, loserate: float) -> Tuple[float, float]:
-
-        expectancy = (
-            (winrate * mean_winning_profit) -
-            (loserate * mean_losing_profit)
-        )
-
-        expectancy_ratio = float('inf')
-        if mean_losing_profit > 0:
-            expectancy_ratio = (
-                ((1 + (mean_winning_profit / mean_losing_profit)) * winrate) - 1
-            )
-
-        return expectancy, expectancy_ratio
-
     def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict:
         """ Returns current account balance per crypto """
         currencies: List[Dict] = []

freqtrade/rpc/webhook.py

@@ -34,6 +34,7 @@ class Webhook(RPCHandler):
         self._format = self._config['webhook'].get('format', 'form')
         self._retries = self._config['webhook'].get('retries', 0)
         self._retry_delay = self._config['webhook'].get('retry_delay', 0.1)
+        self._timeout = self._config['webhook'].get('timeout', 10)
 
     def cleanup(self) -> None:
         """
@@ -107,12 +108,13 @@ class Webhook(RPCHandler):
 
             try:
                 if self._format == 'form':
-                    response = post(self._url, data=payload)
+                    response = post(self._url, data=payload, timeout=self._timeout)
                 elif self._format == 'json':
-                    response = post(self._url, json=payload)
+                    response = post(self._url, json=payload, timeout=self._timeout)
                 elif self._format == 'raw':
                     response = post(self._url, data=payload['data'],
-                                    headers={'Content-Type': 'text/plain'})
+                                    headers={'Content-Type': 'text/plain'},
+                                    timeout=self._timeout)
                 else:
                     raise NotImplementedError(f'Unknown format: {self._format}')
 

requirements-freqai.txt

@@ -6,7 +6,7 @@
 scikit-learn==1.1.3
 joblib==1.3.1
 catboost==1.2; 'arm' not in platform_machine
-lightgbm==3.3.5
+lightgbm==4.0.0
 xgboost==1.7.6
 tensorboard==2.13.0
 datasieve==0.1.7

tests/data/test_btanalysis.py

@@ -343,12 +343,24 @@ def test_calculate_expectancy(testdatadir):
     filename = testdatadir / "backtest_results/backtest-result.json"
     bt_data = load_backtest_data(filename)
 
-    expectancy = calculate_expectancy(DataFrame())
+    expectancy, expectancy_ratio = calculate_expectancy(DataFrame())
     assert expectancy == 0.0
+    assert expectancy_ratio == 100
 
-    expectancy = calculate_expectancy(bt_data)
+    expectancy, expectancy_ratio = calculate_expectancy(bt_data)
     assert isinstance(expectancy, float)
-    assert pytest.approx(expectancy) == 0.07151374226574791
+    assert isinstance(expectancy_ratio, float)
+    assert pytest.approx(expectancy) == 5.820687070932315e-06
+    assert pytest.approx(expectancy_ratio) == 0.07151374226574791
+
+    data = {
+        'profit_abs': [100, 200, 50, -150, 300, -100, 80, -30]
+    }
+    df = DataFrame(data)
+    expectancy, expectancy_ratio = calculate_expectancy(df)
+
+    assert pytest.approx(expectancy) == 56.25
+    assert pytest.approx(expectancy_ratio) == 0.60267857
 
 
 def test_calculate_sortino(testdatadir):

tests/rpc/test_rpc.py

@@ -403,7 +403,7 @@ def test_rpc_trade_statistics(default_conf_usdt, ticker, fee, mocker) -> None:
     assert res['latest_trade_date'] == ''
     assert res['latest_trade_timestamp'] == 0
     assert res['expectancy'] == 0
-    assert res['expectancy_ratio'] == float('inf')
+    assert res['expectancy_ratio'] == 100
 
     # Create some test data
     create_mock_trades_usdt(fee)

tests/rpc/test_rpc_apiserver.py

@@ -846,7 +846,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets):
          'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07,
          'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0,
          'profit_factor': None, 'winrate': 1.0, 'expectancy': 0.0003695635,
-         'expectancy_ratio': None, 'trading_volume': 91.074,
+         'expectancy_ratio': 100, 'trading_volume': 91.074,
          }
     ),
     (

tests/rpc/test_rpc_webhook.py

@@ -381,7 +381,7 @@ def test__send_msg(default_conf, mocker, caplog):
     webhook._send_msg(msg)
 
     assert post.call_count == 1
-    assert post.call_args[1] == {'data': msg}
+    assert post.call_args[1] == {'data': msg, 'timeout': 10}
     assert post.call_args[0] == (default_conf['webhook']['url'], )
 
     post = MagicMock(side_effect=RequestException)
@@ -399,7 +399,7 @@ def test__send_msg_with_json_format(default_conf, mocker, caplog):
     mocker.patch("freqtrade.rpc.webhook.post", post)
     webhook._send_msg(msg)
 
-    assert post.call_args[1] == {'json': msg}
+    assert post.call_args[1] == {'json': msg, 'timeout': 10}
 
 
 def test__send_msg_with_raw_format(default_conf, mocker, caplog):
@@ -411,7 +411,11 @@ def test__send_msg_with_raw_format(default_conf, mocker, caplog):
     mocker.patch("freqtrade.rpc.webhook.post", post)
     webhook._send_msg(msg)
 
-    assert post.call_args[1] == {'data': msg['data'], 'headers': {'Content-Type': 'text/plain'}}
+    assert post.call_args[1] == {
+        'data': msg['data'],
+        'headers': {'Content-Type': 'text/plain'},
+        'timeout': 10
+    }
 
 
 def test_send_msg_discord(default_conf, mocker):