From 82e30c8519042187a1c7953ec8387e9b946b48d7 Mon Sep 17 00:00:00 2001 From: KingND <81396266+KingND@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:03:17 -0400 Subject: [PATCH 1/7] feat: if a biased_indicator starting with & appears in a lookahead-analysis, caption the table with a note that freqai targets appearing here can be ignored --- .../optimize/analysis/lookahead_helpers.py | 24 +++- tests/optimize/test_lookahead_analysis.py | 114 ++++++++++++++++++ 2 files changed, 135 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/analysis/lookahead_helpers.py b/freqtrade/optimize/analysis/lookahead_helpers.py index a8fb1cd35..aff2083bf 100644 --- a/freqtrade/optimize/analysis/lookahead_helpers.py +++ b/freqtrade/optimize/analysis/lookahead_helpers.py @@ -19,7 +19,9 @@ logger = logging.getLogger(__name__) class LookaheadAnalysisSubFunctions: @staticmethod def text_table_lookahead_analysis_instances( - config: Dict[str, Any], lookahead_instances: List[LookaheadAnalysis] + config: Dict[str, Any], + lookahead_instances: List[LookaheadAnalysis], + caption: str | None = None, ): headers = [ "filename", @@ -65,7 +67,12 @@ class LookaheadAnalysisSubFunctions: ] ) - print_rich_table(data, headers, summary="Lookahead Analysis") + print_rich_table( + data, + headers, + summary="Lookahead Analysis", + table_kwargs={"caption": caption} + ) return data @staticmethod @@ -239,8 +246,19 @@ class LookaheadAnalysisSubFunctions: # report the results if lookaheadAnalysis_instances: + caption: str | None = None + if any([ + any([ + indicator.startswith("&") + for indicator in inst.current_analysis.false_indicators + ]) for inst in lookaheadAnalysis_instances + ]): + caption = ( + "Any indicators in 'biased_indicators' which are used within " + "set_freqai_targets() can be ignored." + ) LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( - config, lookaheadAnalysis_instances + config, lookaheadAnalysis_instances, caption=caption ) if config.get("lookahead_analysis_exportfilename") is not None: LookaheadAnalysisSubFunctions.export_to_csv(config, lookaheadAnalysis_instances) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index f7d38b24b..387587afb 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -133,6 +133,72 @@ def test_lookahead_helper_start(lookahead_conf, mocker) -> None: text_table_mock.reset_mock() +@pytest.mark.parametrize( + "indicators, expected_caption_text", + [ + ( + ["&indicator1", "indicator2"], + "Any indicators in 'biased_indicators' which are used " + "within set_freqai_targets() can be ignored." + ), + ( + ["indicator1", "&indicator2"], + "Any indicators in 'biased_indicators' which are used " + "within set_freqai_targets() can be ignored." + ), + ( + ["&indicator1", "&indicator2"], + "Any indicators in 'biased_indicators' which are used " + "within set_freqai_targets() can be ignored." + ), + ( + ["indicator1", "indicator2"], + None + ), + ( + [], + None + ) + ], + ids=( + "First of two biased indicators starts with '&'", + "Second of two biased indicators starts with '&'", + "Both biased indicators start with '&'", + "No biased indicators start with '&'", + "Empty biased indicators list", + ) +) +def test_lookahead_helper_start__caption_based_on_indicators( + indicators, + expected_caption_text, + lookahead_conf, + mocker +): + """Test that the table caption is only populated if a biased_indicator starts with '&'.""" + + single_mock = MagicMock() + lookahead_analysis = LookaheadAnalysis( + lookahead_conf, + {"name": "strategy_test_v3_with_lookahead_bias"}, + ) + lookahead_analysis.current_analysis.false_indicators = indicators + single_mock.return_value = lookahead_analysis + text_table_mock = MagicMock() + mocker.patch.multiple( + "freqtrade.optimize.analysis.lookahead_helpers.LookaheadAnalysisSubFunctions", + initialize_single_lookahead_analysis=single_mock, + text_table_lookahead_analysis_instances=text_table_mock, + ) + + LookaheadAnalysisSubFunctions.start(lookahead_conf) + + text_table_mock.assert_called_once_with( + lookahead_conf, + [lookahead_analysis], + caption=expected_caption_text + ) + + def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf): analysis = Analysis() analysis.has_bias = True @@ -199,6 +265,54 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf assert len(data) == 3 + +@pytest.mark.parametrize( + "caption", + [ + "", + "A test caption", + None, + False, + ], + ids=( + "Pass empty string", + "Pass non-empty string", + "Pass None", + "Don't pass caption", + ) +) +def test_lookahead_helper_text_table_lookahead_analysis_instances__caption( + caption, + lookahead_conf, + mocker, +): + """Test that the caption is passed in the table kwargs when calling print_rich_table().""" + + print_rich_table_mock = MagicMock() + mocker.patch( + "freqtrade.optimize.analysis.lookahead_helpers.print_rich_table", + print_rich_table_mock, + ) + lookahead_analysis = LookaheadAnalysis( + lookahead_conf, + { + "name": "strategy_test_v3_with_lookahead_bias", + "location": Path(lookahead_conf["strategy_path"], f"{lookahead_conf['strategy']}.py"), + } + ) + kwargs = {} + if caption is not False: + kwargs["caption"] = caption + + LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + lookahead_conf, [lookahead_analysis], **kwargs + ) + + assert print_rich_table_mock.call_args[-1]["table_kwargs"]["caption"] == ( + caption if caption is not False else None + ) + + def test_lookahead_helper_export_to_csv(lookahead_conf): import pandas as pd From 5f52fc4338c59029f4d176d2103a4bf40ab5bf74 Mon Sep 17 00:00:00 2001 From: KingND <81396266+KingND@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:17:53 -0400 Subject: [PATCH 2/7] feat: update lookahead-analysis doc caveats to include info regarding the false positive on FreqAI targets --- docs/lookahead-analysis.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/lookahead-analysis.md b/docs/lookahead-analysis.md index 90ba7041a..12dc355fa 100644 --- a/docs/lookahead-analysis.md +++ b/docs/lookahead-analysis.md @@ -101,3 +101,4 @@ This could lead to a false-negative (the strategy will then be reported as non-b - `lookahead-analysis` has access to everything that backtesting has too. Please don't provoke any configs like enabling position stacking. If you decide to do so, then make doubly sure that you won't ever run out of `max_open_trades` amount and neither leftover money in your wallet. +- `biased_indicators` will falsely flag FreqAI target indicators defined in `set_freqai_targets()` as biased. These are not biased and can safely be ignored. From bb9f64027af88abaca71c9fb72e94fd1c8a8ff49 Mon Sep 17 00:00:00 2001 From: KingND <81396266+KingND@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:32:13 -0400 Subject: [PATCH 3/7] chore: improve language in docs --- docs/lookahead-analysis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lookahead-analysis.md b/docs/lookahead-analysis.md index 12dc355fa..1cdf9aaf0 100644 --- a/docs/lookahead-analysis.md +++ b/docs/lookahead-analysis.md @@ -101,4 +101,4 @@ This could lead to a false-negative (the strategy will then be reported as non-b - `lookahead-analysis` has access to everything that backtesting has too. Please don't provoke any configs like enabling position stacking. If you decide to do so, then make doubly sure that you won't ever run out of `max_open_trades` amount and neither leftover money in your wallet. -- `biased_indicators` will falsely flag FreqAI target indicators defined in `set_freqai_targets()` as biased. These are not biased and can safely be ignored. +- In the results table, the `biased_indicators` column will falsely flag FreqAI target indicators defined in `set_freqai_targets()` as biased. These are not biased and can safely be ignored. From c6c65b1799c99e120c4a58a024ed838cae02534d Mon Sep 17 00:00:00 2001 From: KingND <81396266+KingND@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:38:56 -0400 Subject: [PATCH 4/7] chore: flake8 --- tests/optimize/test_lookahead_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 387587afb..eb64ad42d 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -265,7 +265,6 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf assert len(data) == 3 - @pytest.mark.parametrize( "caption", [ From 53cab5074b0f94800e62c400dee7750b9121f986 Mon Sep 17 00:00:00 2001 From: KingND <81396266+KingND@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:42:44 -0400 Subject: [PATCH 5/7] chore: refactor and cleanup tests --- tests/optimize/test_lookahead_analysis.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index eb64ad42d..dd036cdc9 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -13,6 +13,12 @@ from freqtrade.optimize.analysis.lookahead_helpers import LookaheadAnalysisSubFu from tests.conftest import EXMS, get_args, log_has_re, patch_exchange +IGNORE_BIASED_INDICATORS_CAPTION = ( + "Any indicators in 'biased_indicators' which are used within " + "set_freqai_targets() can be ignored." +) + + @pytest.fixture def lookahead_conf(default_conf_usdt, tmp_path): default_conf_usdt["user_data_dir"] = tmp_path @@ -138,18 +144,15 @@ def test_lookahead_helper_start(lookahead_conf, mocker) -> None: [ ( ["&indicator1", "indicator2"], - "Any indicators in 'biased_indicators' which are used " - "within set_freqai_targets() can be ignored." + IGNORE_BIASED_INDICATORS_CAPTION, ), ( ["indicator1", "&indicator2"], - "Any indicators in 'biased_indicators' which are used " - "within set_freqai_targets() can be ignored." + IGNORE_BIASED_INDICATORS_CAPTION, ), ( ["&indicator1", "&indicator2"], - "Any indicators in 'biased_indicators' which are used " - "within set_freqai_targets() can be ignored." + IGNORE_BIASED_INDICATORS_CAPTION, ), ( ["indicator1", "indicator2"], @@ -158,7 +161,7 @@ def test_lookahead_helper_start(lookahead_conf, mocker) -> None: ( [], None - ) + ), ], ids=( "First of two biased indicators starts with '&'", @@ -166,7 +169,7 @@ def test_lookahead_helper_start(lookahead_conf, mocker) -> None: "Both biased indicators start with '&'", "No biased indicators start with '&'", "Empty biased indicators list", - ) + ), ) def test_lookahead_helper_start__caption_based_on_indicators( indicators, @@ -278,7 +281,7 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf "Pass non-empty string", "Pass None", "Don't pass caption", - ) + ), ) def test_lookahead_helper_text_table_lookahead_analysis_instances__caption( caption, From 69678574d4e3421ed3fe697a58b9624091f03242 Mon Sep 17 00:00:00 2001 From: KingND <81396266+KingND@users.noreply.github.com> Date: Sat, 7 Sep 2024 16:01:52 -0400 Subject: [PATCH 6/7] fix: support python 3.9 union type hinting --- freqtrade/optimize/analysis/lookahead_helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/analysis/lookahead_helpers.py b/freqtrade/optimize/analysis/lookahead_helpers.py index aff2083bf..b3cba9ed4 100644 --- a/freqtrade/optimize/analysis/lookahead_helpers.py +++ b/freqtrade/optimize/analysis/lookahead_helpers.py @@ -1,7 +1,7 @@ import logging import time from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Union import pandas as pd from rich.text import Text @@ -21,7 +21,7 @@ class LookaheadAnalysisSubFunctions: def text_table_lookahead_analysis_instances( config: Dict[str, Any], lookahead_instances: List[LookaheadAnalysis], - caption: str | None = None, + caption: Union[str, None] = None, ): headers = [ "filename", @@ -246,7 +246,7 @@ class LookaheadAnalysisSubFunctions: # report the results if lookaheadAnalysis_instances: - caption: str | None = None + caption: Union[str, None] = None if any([ any([ indicator.startswith("&") From f970454cb49d6d9f213e4114a960de2f4c2516f6 Mon Sep 17 00:00:00 2001 From: KingND <81396266+KingND@users.noreply.github.com> Date: Sun, 8 Sep 2024 13:57:48 -0400 Subject: [PATCH 7/7] chore: ruff format --- .../optimize/analysis/lookahead_helpers.py | 22 ++++++++++--------- tests/optimize/test_lookahead_analysis.py | 21 +++++------------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/freqtrade/optimize/analysis/lookahead_helpers.py b/freqtrade/optimize/analysis/lookahead_helpers.py index b3cba9ed4..730f9fd72 100644 --- a/freqtrade/optimize/analysis/lookahead_helpers.py +++ b/freqtrade/optimize/analysis/lookahead_helpers.py @@ -68,10 +68,7 @@ class LookaheadAnalysisSubFunctions: ) print_rich_table( - data, - headers, - summary="Lookahead Analysis", - table_kwargs={"caption": caption} + data, headers, summary="Lookahead Analysis", table_kwargs={"caption": caption} ) return data @@ -247,12 +244,17 @@ class LookaheadAnalysisSubFunctions: # report the results if lookaheadAnalysis_instances: caption: Union[str, None] = None - if any([ - any([ - indicator.startswith("&") - for indicator in inst.current_analysis.false_indicators - ]) for inst in lookaheadAnalysis_instances - ]): + if any( + [ + any( + [ + indicator.startswith("&") + for indicator in inst.current_analysis.false_indicators + ] + ) + for inst in lookaheadAnalysis_instances + ] + ): caption = ( "Any indicators in 'biased_indicators' which are used within " "set_freqai_targets() can be ignored." diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index dd036cdc9..67c83762a 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -154,14 +154,8 @@ def test_lookahead_helper_start(lookahead_conf, mocker) -> None: ["&indicator1", "&indicator2"], IGNORE_BIASED_INDICATORS_CAPTION, ), - ( - ["indicator1", "indicator2"], - None - ), - ( - [], - None - ), + (["indicator1", "indicator2"], None), + ([], None), ], ids=( "First of two biased indicators starts with '&'", @@ -172,10 +166,7 @@ def test_lookahead_helper_start(lookahead_conf, mocker) -> None: ), ) def test_lookahead_helper_start__caption_based_on_indicators( - indicators, - expected_caption_text, - lookahead_conf, - mocker + indicators, expected_caption_text, lookahead_conf, mocker ): """Test that the table caption is only populated if a biased_indicator starts with '&'.""" @@ -196,9 +187,7 @@ def test_lookahead_helper_start__caption_based_on_indicators( LookaheadAnalysisSubFunctions.start(lookahead_conf) text_table_mock.assert_called_once_with( - lookahead_conf, - [lookahead_analysis], - caption=expected_caption_text + lookahead_conf, [lookahead_analysis], caption=expected_caption_text ) @@ -300,7 +289,7 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances__caption( { "name": "strategy_test_v3_with_lookahead_bias", "location": Path(lookahead_conf["strategy_path"], f"{lookahead_conf['strategy']}.py"), - } + }, ) kwargs = {} if caption is not False: