Merge pull request #9014 from hippocritical/develop

bugfixes and false-positives for lookahead-analysis

docs/lookahead-analysis.md

@@ -21,7 +21,10 @@ It also supports the lookahead-analysis of freqai strategies.
 
 - `--cache` is forced to "none".
 - `--max-open-trades` is forced to be at least equal to the number of pairs.
-- `--dry-run-wallet` is forced to be basically infinite.
+- `--dry-run-wallet` is forced to be basically infinite (1 billion).
+- `--stake-amount` is forced to be a static 10000 (10k).
+
+Those are set to avoid users accidentally generating false positives.
 
 ## Lookahead-analysis command reference
 

freqtrade/optimize/lookahead_analysis.py

@@ -48,6 +48,7 @@ class LookaheadAnalysis:
         self.entry_varHolders: List[VarHolder] = []
         self.exit_varHolders: List[VarHolder] = []
         self.exchange: Optional[Any] = None
+        self._fee = None
 
         # pull variables the scope of the lookahead_analysis-instance
         self.local_config = deepcopy(config)
@@ -145,8 +146,13 @@ class LookaheadAnalysis:
                                             str(self.dt_to_timestamp(varholder.to_dt)))
         prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load
 
+        if self._fee is not None:
+            # Don't re-calculate fee per pair, as fee might differ per pair.
+            prepare_data_config['fee'] = self._fee
+
         backtesting = Backtesting(prepare_data_config, self.exchange)
         self.exchange = backtesting.exchange
+        self._fee = backtesting.fee
         backtesting._set_strategy(backtesting.strategylist[0])
 
         varholder.data, varholder.timerange = backtesting.load_bt_data()
@@ -198,7 +204,7 @@ class LookaheadAnalysis:
         self.prepare_data(exit_varHolder, [result_row['pair']])
 
     # now we analyze a full trade of full_varholder and look for analyze its bias
-    def analyze_row(self, idx, result_row):
+    def analyze_row(self, idx: int, result_row):
         # if force-sold, ignore this signal since here it will unconditionally exit.
         if result_row.close_date == self.dt_to_timestamp(self.full_varHolder.to_dt):
             return
@@ -209,12 +215,16 @@ class LookaheadAnalysis:
         # fill entry_varHolder and exit_varHolder
         self.fill_entry_and_exit_varHolders(result_row)
 
+        # this will trigger a logger-message
+        buy_or_sell_biased: bool = False
+
         # register if buy signal is broken
         if not self.report_signal(
                 self.entry_varHolders[idx].result,
                 "open_date",
                 self.entry_varHolders[idx].compared_dt):
             self.current_analysis.false_entry_signals += 1
+            buy_or_sell_biased = True
 
         # register if buy or sell signal is broken
         if not self.report_signal(
@@ -222,6 +232,13 @@ class LookaheadAnalysis:
                 "close_date",
                 self.exit_varHolders[idx].compared_dt):
             self.current_analysis.false_exit_signals += 1
+            buy_or_sell_biased = True
+
+        if buy_or_sell_biased:
+            logger.info(f"found lookahead-bias in trade "
+                        f"pair: {result_row['pair']}, "
+                        f"timerange:{result_row['open_date']} - {result_row['close_date']}, "
+                        f"idx: {idx}")
 
         # check if the indicators themselves contain biased data
         self.analyze_indicators(self.full_varHolder, self.entry_varHolders[idx], result_row['pair'])
@@ -251,9 +268,33 @@ class LookaheadAnalysis:
         # starting from the same datetime to avoid miss-reports of bias
         for idx, result_row in self.full_varHolder.result['results'].iterrows():
             if self.current_analysis.total_signals == self.targeted_trade_amount:
+                logger.info(f"Found targeted trade amount = {self.targeted_trade_amount} signals.")
                 break
+            if found_signals < self.minimum_trade_amount:
+                logger.info(f"only found {found_signals} "
+                            f"which is smaller than "
+                            f"minimum trade amount = {self.minimum_trade_amount}. "
+                            f"Exiting this lookahead-analysis")
+                return None
+            if "force_exit" in result_row['exit_reason']:
+                logger.info("found force-exit in pair: {result_row['pair']}, "
+                            f"timerange:{result_row['open_date']}-{result_row['close_date']}, "
+                            f"idx: {idx}, skipping this one to avoid a false-positive.")
+
+                # just to keep the IDs of both full, entry and exit varholders the same
+                # to achieve a better debugging experience
+                self.entry_varHolders.append(VarHolder())
+                self.exit_varHolders.append(VarHolder())
+                continue
+
             self.analyze_row(idx, result_row)
 
+        if len(self.entry_varHolders) < self.minimum_trade_amount:
+            logger.info(f"only found {found_signals} after skipping forced exits "
+                        f"which is smaller than "
+                        f"minimum trade amount = {self.minimum_trade_amount}. "
+                        f"Exiting this lookahead-analysis")
+
         # Restore verbosity, so it's not too quiet for the next strategy
         restore_verbosity_for_bias_tester()
         # check and report signals

freqtrade/optimize/lookahead_analysis_helpers.py

@@ -137,6 +137,19 @@ class LookaheadAnalysisSubFunctions:
                         'just to avoid false positives')
             config['dry_run_wallet'] = min_dry_run_wallet
 
+        if 'timerange' not in config:
+            # setting a timerange is enforced here
+            raise OperationalException(
+                "Please set a timerange. "
+                "Usually a few months are enough depending on your needs and strategy."
+            )
+        # fix stake_amount to 10k.
+        # in a combination with a wallet size of 1 billion it should always be able to trade
+        # no matter if they use custom_stake_amount as a small percentage of wallet size
+        # or fixate custom_stake_amount to a certain value.
+        logger.info('fixing stake_amount to 10k')
+        config['stake_amount'] = 10000
+
         # enforce cache to be 'none', shift it to 'none' if not already
         # (since the default value is 'day')
         if config.get('backtest_cache') is None:

tests/optimize/test_lookahead_analysis.py

@@ -17,6 +17,8 @@ from tests.conftest import EXMS, get_args, log_has_re, patch_exchange
 def lookahead_conf(default_conf_usdt):
     default_conf_usdt['minimum_trade_amount'] = 10
     default_conf_usdt['targeted_trade_amount'] = 20
+    default_conf_usdt['timerange'] = '20220101-20220501'
+
     default_conf_usdt['strategy_path'] = str(
         Path(__file__).parent.parent / "strategy/strats/lookahead_bias")
     default_conf_usdt['strategy'] = 'strategy_test_v3_with_lookahead_bias'
@@ -43,7 +45,9 @@ def test_start_lookahead_analysis(mocker):
         "--pairs",
         "UNITTEST/BTC",
         "--max-open-trades",
-        "1"
+        "1",
+        "--timerange",
+        "20220101-20220201"
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -72,6 +76,24 @@ def test_start_lookahead_analysis(mocker):
                        match=r"Targeted trade amount can't be smaller than minimum trade amount.*"):
         start_lookahead_analysis(pargs)
 
+    # Missing timerange
+    args = [
+        "lookahead-analysis",
+        "--strategy",
+        "strategy_test_v3_with_lookahead_bias",
+        "--strategy-path",
+        str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"),
+        "--pairs",
+        "UNITTEST/BTC",
+        "--max-open-trades",
+        "1",
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    with pytest.raises(OperationalException,
+                       match=r"Please set a timerange\..*"):
+        start_lookahead_analysis(pargs)
+
 
 def test_lookahead_helper_invalid_config(lookahead_conf) -> None:
     conf = deepcopy(lookahead_conf)