From 3cdd06f5627b7baee7b5ec3ead09e036080e9b66 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 13 Sep 2021 19:32:51 +0200
Subject: [PATCH 01/13] Add PeriodicCache

---
 freqtrade/configuration/PeriodicCache.py | 19 +++++++++++++++
 freqtrade/configuration/__init__.py      |  1 +
 requirements-dev.txt                     |  2 ++
 tests/test_periodiccache.py              | 31 ++++++++++++++++++++++++
 4 files changed, 53 insertions(+)
 create mode 100644 freqtrade/configuration/PeriodicCache.py
 create mode 100644 tests/test_periodiccache.py

diff --git a/freqtrade/configuration/PeriodicCache.py b/freqtrade/configuration/PeriodicCache.py
new file mode 100644
index 000000000..25c0c47f3
--- /dev/null
+++ b/freqtrade/configuration/PeriodicCache.py
@@ -0,0 +1,19 @@
+from datetime import datetime, timezone
+
+from cachetools.ttl import TTLCache
+
+
+class PeriodicCache(TTLCache):
+    """
+    Special cache that expires at "straight" times
+    A timer with ttl of 3600 (1h) will expire at every full hour (:00).
+    """
+
+    def __init__(self, maxsize, ttl, getsizeof=None):
+        def local_timer():
+            ts = datetime.now(timezone.utc).timestamp()
+            offset = (ts % ttl)
+            return ts - offset
+
+        # Init with smlight offset
+        super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof)
diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py
index 607f9cdef..b1b268092 100644
--- a/freqtrade/configuration/__init__.py
+++ b/freqtrade/configuration/__init__.py
@@ -1,5 +1,6 @@
 # flake8: noqa: F401
 
+from freqtrade.configuration.PeriodicCache import PeriodicCache
 from freqtrade.configuration.check_exchange import check_exchange, remove_credentials
 from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.config_validation import validate_config_consistency
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 34d5607f3..4859e1cc6 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -14,6 +14,8 @@ pytest-cov==2.12.1
 pytest-mock==3.6.1
 pytest-random-order==1.0.4
 isort==5.9.3
+# For datetime mocking
+time-machine==2.4.0
 
 # Convert jupyter notebooks to markdown documents
 nbconvert==6.1.0
diff --git a/tests/test_periodiccache.py b/tests/test_periodiccache.py
new file mode 100644
index 000000000..ff9b53684
--- /dev/null
+++ b/tests/test_periodiccache.py
@@ -0,0 +1,31 @@
+from freqtrade.configuration import PeriodicCache
+import time_machine
+
+
+def test_ttl_cache():
+
+    with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
+
+        cache = PeriodicCache(5, ttl=60)
+        cache1h = PeriodicCache(5, ttl=3600)
+
+        assert cache.timer() == 1630472400.0
+        cache['a'] = 1235
+        cache1h['a'] = 555123
+        assert 'a' in cache
+        assert 'a' in cache1h
+
+        t.move_to("2021-09-01 05:00:59 +00:00")
+        assert 'a' in cache
+        assert 'a' in cache1h
+
+        # Cache expired
+        t.move_to("2021-09-01 05:01:00 +00:00")
+        assert 'a' not in cache
+        assert 'a' in cache1h
+
+        t.move_to("2021-09-01 05:59:59 +00:00")
+        assert 'a' in cache1h
+
+        t.move_to("2021-09-01 06:00:00 +00:00")
+        assert 'a' not in cache1h

From 8afb3c4b70c3458772953c8be23976bd90a264ba Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 13 Sep 2021 19:33:28 +0200
Subject: [PATCH 02/13] Move AgeFilter cache to instance level

---
 freqtrade/plugins/pairlist/AgeFilter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index dc5cab31e..32fef5fb0 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -18,14 +18,14 @@ logger = logging.getLogger(__name__)
 
 class AgeFilter(IPairList):
 
-    # Checked symbols cache (dictionary of ticker symbol => timestamp)
-    _symbolsChecked: Dict[str, int] = {}
-
     def __init__(self, exchange, pairlistmanager,
                  config: Dict[str, Any], pairlistconfig: Dict[str, Any],
                  pairlist_pos: int) -> None:
         super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
 
+        # Checked symbols cache (dictionary of ticker symbol => timestamp)
+        self._symbolsChecked: Dict[str, int] = {}
+
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
         self._max_days_listed = pairlistconfig.get('max_days_listed', None)
 

From c9ba52d7321b2cae17776c66723c837a75c3bfdf Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Sep 2021 06:30:18 +0200
Subject: [PATCH 03/13] Expire cached pairs in age-filter once per day

---
 freqtrade/plugins/pairlist/AgeFilter.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 32fef5fb0..1fba00123 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -11,6 +11,7 @@ from pandas import DataFrame
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import plural
 from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.configuration import PeriodicCache
 
 
 logger = logging.getLogger(__name__)
@@ -25,6 +26,7 @@ class AgeFilter(IPairList):
 
         # Checked symbols cache (dictionary of ticker symbol => timestamp)
         self._symbolsChecked: Dict[str, int] = {}
+        self._too_young_pairs = PeriodicCache(maxsize=1000, ttl=86_400)
 
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
         self._max_days_listed = pairlistconfig.get('max_days_listed', None)
@@ -69,10 +71,12 @@ class AgeFilter(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new allowlist
         """
-        needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked]
+        needed_pairs = [
+            (p, '1d') for p in pairlist
+            if p not in self._symbolsChecked and p not in self._too_young_pairs]
         if not needed_pairs:
             return pairlist
-
+        logger.info(f"needed pairs {needed_pairs}")
         since_days = -(
             self._max_days_listed if self._max_days_listed else self._min_days_listed
         ) - 1
@@ -118,5 +122,6 @@ class AgeFilter(IPairList):
                     " or more than "
                     f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
                 ) if self._max_days_listed else ''), logger.info)
+                self._too_young_pairs[pair] = arrow.utcnow().int_timestamp * 1000
                 return False
         return False

From 3ce5197e8d07471e75be14df435c3d291b3b69c4 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Sep 2021 06:45:26 +0200
Subject: [PATCH 04/13] Add Tests for AgeFilter caching

closes #5552
---
 freqtrade/configuration/__init__.py     |  2 +-
 freqtrade/plugins/pairlist/AgeFilter.py | 11 ++--
 tests/plugins/test_pairlist.py          | 69 ++++++++++++++++---------
 tests/test_periodiccache.py             |  3 +-
 4 files changed, 54 insertions(+), 31 deletions(-)

diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py
index b1b268092..dccbb14b3 100644
--- a/freqtrade/configuration/__init__.py
+++ b/freqtrade/configuration/__init__.py
@@ -1,8 +1,8 @@
 # flake8: noqa: F401
 
-from freqtrade.configuration.PeriodicCache import PeriodicCache
 from freqtrade.configuration.check_exchange import check_exchange, remove_credentials
 from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.configuration.configuration import Configuration
+from freqtrade.configuration.PeriodicCache import PeriodicCache
 from freqtrade.configuration.timerange import TimeRange
diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 1fba00123..c43bd0c4c 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -8,10 +8,10 @@ from typing import Any, Dict, List, Optional
 import arrow
 from pandas import DataFrame
 
+from freqtrade.configuration import PeriodicCache
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import plural
 from freqtrade.plugins.pairlist.IPairList import IPairList
-from freqtrade.configuration import PeriodicCache
 
 
 logger = logging.getLogger(__name__)
@@ -26,7 +26,7 @@ class AgeFilter(IPairList):
 
         # Checked symbols cache (dictionary of ticker symbol => timestamp)
         self._symbolsChecked: Dict[str, int] = {}
-        self._too_young_pairs = PeriodicCache(maxsize=1000, ttl=86_400)
+        self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400)
 
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
         self._max_days_listed = pairlistconfig.get('max_days_listed', None)
@@ -73,9 +73,10 @@ class AgeFilter(IPairList):
         """
         needed_pairs = [
             (p, '1d') for p in pairlist
-            if p not in self._symbolsChecked and p not in self._too_young_pairs]
+            if p not in self._symbolsChecked and p not in self._symbolsCheckFailed]
         if not needed_pairs:
-            return pairlist
+            # Remove pairs that have been removed before
+            return [p for p in pairlist if p not in self._symbolsCheckFailed]
         logger.info(f"needed pairs {needed_pairs}")
         since_days = -(
             self._max_days_listed if self._max_days_listed else self._min_days_listed
@@ -122,6 +123,6 @@ class AgeFilter(IPairList):
                     " or more than "
                     f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
                 ) if self._max_days_listed else ''), logger.info)
-                self._too_young_pairs[pair] = arrow.utcnow().int_timestamp * 1000
+                self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000
                 return False
         return False
diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py
index 5f0701a22..3cdf6d341 100644
--- a/tests/plugins/test_pairlist.py
+++ b/tests/plugins/test_pairlist.py
@@ -4,6 +4,7 @@ import time
 from unittest.mock import MagicMock, PropertyMock
 
 import pytest
+import time_machine
 
 from freqtrade.constants import AVAILABLE_PAIRLISTS
 from freqtrade.exceptions import OperationalException
@@ -815,32 +816,52 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick
 
 
 def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history):
-    ohlcv_data = {
-        ('ETH/BTC', '1d'): ohlcv_history,
-        ('TKN/BTC', '1d'): ohlcv_history,
-        ('LTC/BTC', '1d'): ohlcv_history,
-    }
-    mocker.patch.multiple('freqtrade.exchange.Exchange',
-                          markets=PropertyMock(return_value=markets),
-                          exchange_has=MagicMock(return_value=True),
-                          get_tickers=tickers
-                          )
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
-    )
+    with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
+        ohlcv_data = {
+            ('ETH/BTC', '1d'): ohlcv_history,
+            ('TKN/BTC', '1d'): ohlcv_history,
+            ('LTC/BTC', '1d'): ohlcv_history,
+            ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]],
+        }
+        mocker.patch.multiple(
+            'freqtrade.exchange.Exchange',
+            markets=PropertyMock(return_value=markets),
+            exchange_has=MagicMock(return_value=True),
+            get_tickers=tickers,
+            refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
+        )
 
-    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
-    assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
-    freqtrade.pairlists.refresh_pairlist()
-    assert len(freqtrade.pairlists.whitelist) == 3
-    assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
+        freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 3
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
 
-    previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count
-    freqtrade.pairlists.refresh_pairlist()
-    assert len(freqtrade.pairlists.whitelist) == 3
-    # Called once for XRP/BTC
-    assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1
+        previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 3
+        # Call to XRP/BTC cached
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count
+        # Move to next day
+        t.move_to("2021-09-02 01:00:00 +00:00")
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 3
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1
+
+        # Move another day with fresh mocks (now the pair is old enough)
+        t.move_to("2021-09-03 01:00:00 +00:00")
+        # Called once for XRP/BTC
+        ohlcv_data = {
+            ('ETH/BTC', '1d'): ohlcv_history,
+            ('TKN/BTC', '1d'): ohlcv_history,
+            ('LTC/BTC', '1d'): ohlcv_history,
+            ('XRP/BTC', '1d'): ohlcv_history,
+        }
+        mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 4
+        # Called once (only for XRP/BTC)
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
 
 
 def test_OffsetFilter_error(mocker, whitelist_conf) -> None:
diff --git a/tests/test_periodiccache.py b/tests/test_periodiccache.py
index ff9b53684..f874f9041 100644
--- a/tests/test_periodiccache.py
+++ b/tests/test_periodiccache.py
@@ -1,6 +1,7 @@
-from freqtrade.configuration import PeriodicCache
 import time_machine
 
+from freqtrade.configuration import PeriodicCache
+
 
 def test_ttl_cache():
 

From 35eda8c8c7c304051cae6b2a22453f56aa0ea4a3 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Sep 2021 07:07:20 +0200
Subject: [PATCH 05/13] Improve agefilter test

---
 tests/plugins/test_pairlist.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py
index 3cdf6d341..34770c03d 100644
--- a/tests/plugins/test_pairlist.py
+++ b/tests/plugins/test_pairlist.py
@@ -821,7 +821,6 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o
             ('ETH/BTC', '1d'): ohlcv_history,
             ('TKN/BTC', '1d'): ohlcv_history,
             ('LTC/BTC', '1d'): ohlcv_history,
-            ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]],
         }
         mocker.patch.multiple(
             'freqtrade.exchange.Exchange',
@@ -837,16 +836,28 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o
         assert len(freqtrade.pairlists.whitelist) == 3
         assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
 
-        previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count
         freqtrade.pairlists.refresh_pairlist()
         assert len(freqtrade.pairlists.whitelist) == 3
         # Call to XRP/BTC cached
-        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count
-        # Move to next day
-        t.move_to("2021-09-02 01:00:00 +00:00")
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2
+
+        ohlcv_data = {
+            ('ETH/BTC', '1d'): ohlcv_history,
+            ('TKN/BTC', '1d'): ohlcv_history,
+            ('LTC/BTC', '1d'): ohlcv_history,
+            ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]],
+        }
+        mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
         freqtrade.pairlists.refresh_pairlist()
         assert len(freqtrade.pairlists.whitelist) == 3
-        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
+
+        # Move to next day
+        t.move_to("2021-09-02 01:00:00 +00:00")
+        mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 3
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
 
         # Move another day with fresh mocks (now the pair is old enough)
         t.move_to("2021-09-03 01:00:00 +00:00")

From f7bae81d968a20e815d97d1dc72f86f0573edad2 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Sep 2021 19:56:12 +0200
Subject: [PATCH 06/13] Dataframe should be copied after populate_indicator

Without that, PerformanceWarnings can appear throughout hyperopt which
are unnecessary and missleading for users

closes #5408
---
 freqtrade/strategy/interface.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py
index 91963f223..00ad3faf0 100644
--- a/freqtrade/strategy/interface.py
+++ b/freqtrade/strategy/interface.py
@@ -777,10 +777,11 @@ class IStrategy(ABC, HyperStrategyMixin):
         Does not run advise_buy or advise_sell!
         Used by optimize operations only, not during dry / live runs.
         Using .copy() to get a fresh copy of the dataframe for every strategy run.
+        Also copy on output to avoid PerformanceWarnings pandas 1.3.0 started to show.
         Has positive effects on memory usage for whatever reason - also when
         using only one strategy.
         """
-        return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair})
+        return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}).copy()
                 for pair, pair_data in data.items()}
 
     def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

From 57ea0c322f8018992cb6c4303a43af29da310227 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Sep 2021 20:20:31 +0200
Subject: [PATCH 07/13] Rename indicator_space to buy_indicator_space

---
 docs/hyperopt.md                    | 2 +-
 freqtrade/optimize/hyperopt.py      | 2 +-
 freqtrade/optimize/hyperopt_auto.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/hyperopt.md b/docs/hyperopt.md
index e69b761c4..09d43939a 100644
--- a/docs/hyperopt.md
+++ b/docs/hyperopt.md
@@ -677,7 +677,7 @@ If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace f
 
 These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used.
 
-If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default.
+If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default.
 
 Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps).
 
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 14b155546..d047b7311 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -241,7 +241,7 @@ class Hyperopt:
 
         if HyperoptTools.has_space(self.config, 'buy'):
             logger.debug("Hyperopt has 'buy' space")
-            self.buy_space = self.custom_hyperopt.indicator_space()
+            self.buy_space = self.custom_hyperopt.buy_indicator_space()
 
         if HyperoptTools.has_space(self.config, 'sell'):
             logger.debug("Hyperopt has 'sell' space")
diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py
index 1f11cec80..80fe090c2 100644
--- a/freqtrade/optimize/hyperopt_auto.py
+++ b/freqtrade/optimize/hyperopt_auto.py
@@ -56,7 +56,7 @@ class HyperOptAuto(IHyperOpt):
         else:
             _format_exception_message(category)
 
-    def indicator_space(self) -> List['Dimension']:
+    def buy_indicator_space(self) -> List['Dimension']:
         return self._get_indicator_space('buy')
 
     def sell_indicator_space(self) -> List['Dimension']:

From 90ad1789323e2c3061110f34bd5d3c54d60b65d7 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Sep 2021 21:04:25 +0200
Subject: [PATCH 08/13] Remove verbosity of edge

---
 freqtrade/plugins/pairlist/AgeFilter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index c43bd0c4c..5627d82ce 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -77,7 +77,7 @@ class AgeFilter(IPairList):
         if not needed_pairs:
             # Remove pairs that have been removed before
             return [p for p in pairlist if p not in self._symbolsCheckFailed]
-        logger.info(f"needed pairs {needed_pairs}")
+
         since_days = -(
             self._max_days_listed if self._max_days_listed else self._min_days_listed
         ) - 1

From c0811ae8969799857e987ed7e93d1c1e78dd3a2f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Sep 2021 21:36:53 +0200
Subject: [PATCH 09/13] Add possibility to override estimator from within
 hyperopt

---
 docs/advanced-hyperopt.md                | 32 ++++++++++++++++++++++++
 freqtrade/optimize/hyperopt.py           |  8 ++++--
 freqtrade/optimize/hyperopt_auto.py      |  5 +++-
 freqtrade/optimize/hyperopt_interface.py | 13 +++++++++-
 4 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md
index f2f52b7dd..f5a52ff49 100644
--- a/docs/advanced-hyperopt.md
+++ b/docs/advanced-hyperopt.md
@@ -98,6 +98,38 @@ class MyAwesomeStrategy(IStrategy):
 !!! Note
     All overrides are optional and can be mixed/matched as necessary.
 
+### Overriding Base estimator
+
+You can define your own estimator for Hyperopt by implementing `generate_estimator()` in the Hyperopt subclass.
+
+```python
+class MyAwesomeStrategy(IStrategy):
+    class HyperOpt:
+        def generate_estimator():
+            return "RF"
+
+```
+
+Possible values are either one of "GP", "RF", "ET", "GBRT" (Details can be found in the [scikit-optimize documentation](https://scikit-optimize.github.io/)), or "an instance of a class that inherits from `RegressorMixin` (from sklearn) and where the `predict` method has an optional `return_std` argument, which returns `std(Y | x)` along with `E[Y | x]`".
+
+Some research will be necessary to find additional Regressors.
+
+Example for `ExtraTreesRegressor` ("ET") with additional parameters:
+
+```python
+class MyAwesomeStrategy(IStrategy):
+    class HyperOpt:
+        def generate_estimator():
+            from skopt.learning import ExtraTreesRegressor
+            # Corresponds to "ET" - but allows additional parameters.
+            return ExtraTreesRegressor(n_estimators=100)
+
+```
+
+!!! Note
+    While custom estimators can be provided, it's up to you as User to do research on possible parameters and analyze / understand which ones should be used.
+    If you're unsure about this, best use one of the Defaults (`"ET"` has proven to be the most versatile) without further parameters.
+
 ## Space options
 
 For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types:
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index d047b7311..56d11934a 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -365,10 +365,14 @@ class Hyperopt:
         }
 
     def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
+        estimator = self.custom_hyperopt.generate_estimator()
+        logger.info(f"Using estimator {estimator}.")
+        # TODO: Impact of changing acq_optimizer to "sampling" is unclear
+        # (other than that it fails with other optimizers when using custom sklearn regressors)
         return Optimizer(
             dimensions,
-            base_estimator="ET",
-            acq_optimizer="auto",
+            base_estimator=estimator,
+            acq_optimizer="sampling",
             n_initial_points=INITIAL_POINTS,
             acq_optimizer_kwargs={'n_jobs': cpu_count},
             random_state=self.random_state,
diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py
index 80fe090c2..c1c769c72 100644
--- a/freqtrade/optimize/hyperopt_auto.py
+++ b/freqtrade/optimize/hyperopt_auto.py
@@ -12,7 +12,7 @@ from freqtrade.exceptions import OperationalException
 with suppress(ImportError):
     from skopt.space import Dimension
 
-from freqtrade.optimize.hyperopt_interface import IHyperOpt
+from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
 
 
 def _format_exception_message(space: str) -> str:
@@ -79,3 +79,6 @@ class HyperOptAuto(IHyperOpt):
 
     def trailing_space(self) -> List['Dimension']:
         return self._get_func('trailing_space')()
+
+    def generate_estimator(self) -> EstimatorType:
+        return self._get_func('generate_estimator')()
diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py
index 8fb40f557..53b4f087c 100644
--- a/freqtrade/optimize/hyperopt_interface.py
+++ b/freqtrade/optimize/hyperopt_interface.py
@@ -5,8 +5,9 @@ This module defines the interface to apply for hyperopt
 import logging
 import math
 from abc import ABC
-from typing import Dict, List
+from typing import Dict, List, Union
 
+from sklearn.base import RegressorMixin
 from skopt.space import Categorical, Dimension, Integer
 
 from freqtrade.exchange import timeframe_to_minutes
@@ -17,6 +18,8 @@ from freqtrade.strategy import IStrategy
 
 logger = logging.getLogger(__name__)
 
+EstimatorType = Union[RegressorMixin, str]
+
 
 class IHyperOpt(ABC):
     """
@@ -37,6 +40,14 @@ class IHyperOpt(ABC):
         IHyperOpt.ticker_interval = str(config['timeframe'])  # DEPRECATED
         IHyperOpt.timeframe = str(config['timeframe'])
 
+    def generate_estimator(self) -> EstimatorType:
+        """
+        Return base_estimator.
+        Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class
+        inheriting from RegressorMixin (from sklearn).
+        """
+        return 'ET'
+
     def generate_roi_table(self, params: Dict) -> Dict[int, float]:
         """
         Create a ROI table.

From 5fcb69a0b5463d6db1577ba61c1eccaf656c3b53 Mon Sep 17 00:00:00 2001
From: Sam Germain <sam.germain@usask.ca>
Date: Tue, 14 Sep 2021 23:10:10 -0600
Subject: [PATCH 10/13] Parametrized test_persistence

---
 freqtrade/persistence/models.py |   1 +
 freqtrade/utils/__init__.py     |   3 +
 freqtrade/utils/get_sides.py    |   5 +
 tests/test_persistence.py       | 738 ++++++++++----------------------
 4 files changed, 244 insertions(+), 503 deletions(-)
 create mode 100644 freqtrade/utils/__init__.py
 create mode 100644 freqtrade/utils/get_sides.py

diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py
index a57cf0821..84e402ce5 100644
--- a/freqtrade/persistence/models.py
+++ b/freqtrade/persistence/models.py
@@ -320,6 +320,7 @@ class LocalTrade():
         if self.isolated_liq:
             self.set_isolated_liq(self.isolated_liq)
         self.recalc_open_trade_value()
+        # TODO-lev: Throw exception if on margin and interest_rate is none
 
     def _set_stop_loss(self, stop_loss: float, percent: float):
         """
diff --git a/freqtrade/utils/__init__.py b/freqtrade/utils/__init__.py
new file mode 100644
index 000000000..361a06c38
--- /dev/null
+++ b/freqtrade/utils/__init__.py
@@ -0,0 +1,3 @@
+# flake8: noqa: F401
+
+from freqtrade.utils.get_sides import get_sides
diff --git a/freqtrade/utils/get_sides.py b/freqtrade/utils/get_sides.py
new file mode 100644
index 000000000..9ab97e7b3
--- /dev/null
+++ b/freqtrade/utils/get_sides.py
@@ -0,0 +1,5 @@
+from typing import Tuple
+
+
+def get_sides(is_short: bool) -> Tuple[str, str]:
+    return ("sell", "buy") if is_short else ("buy", "sell")
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 1250e7b92..800e3f541 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -13,6 +13,7 @@ from sqlalchemy import create_engine, inspect, text
 from freqtrade import constants
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.persistence import LocalTrade, Order, Trade, clean_dry_run_db, init_db
+from freqtrade.utils import get_sides
 from tests.conftest import create_mock_trades, create_mock_trades_with_leverage, log_has, log_has_re
 
 
@@ -64,8 +65,10 @@ def test_init_dryrun_db(default_conf, tmpdir):
     assert Path(filename).is_file()
 
 
+@pytest.mark.parametrize('is_short', [False, True])
 @pytest.mark.usefixtures("init_persistence")
-def test_enter_exit_side(fee):
+def test_enter_exit_side(fee, is_short):
+    enter_side, exit_side = get_sides(is_short)
     trade = Trade(
         id=2,
         pair='ADA/USDT',
@@ -77,16 +80,11 @@ def test_enter_exit_side(fee):
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         exchange='binance',
-        is_short=False,
+        is_short=is_short,
         leverage=2.0
     )
-    assert trade.enter_side == 'buy'
-    assert trade.exit_side == 'sell'
-
-    trade.is_short = True
-
-    assert trade.enter_side == 'sell'
-    assert trade.exit_side == 'buy'
+    assert trade.enter_side == enter_side
+    assert trade.exit_side == exit_side
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -170,8 +168,32 @@ def test_set_stop_loss_isolated_liq(fee):
     assert trade.initial_stop_loss == 0.09
 
 
+@pytest.mark.parametrize('exchange,is_short,lev,minutes,rate,interest', [
+    ("binance", False, 3, 10, 0.0005, round(0.0008333333333333334, 8)),
+    ("binance", True, 3, 10, 0.0005, 0.000625),
+    ("binance", False, 3, 295, 0.0005, round(0.004166666666666667, 8)),
+    ("binance", True, 3, 295, 0.0005, round(0.0031249999999999997, 8)),
+    ("binance", False, 3, 295, 0.00025, round(0.0020833333333333333, 8)),
+    ("binance", True, 3, 295, 0.00025, round(0.0015624999999999999, 8)),
+    ("binance", False, 5, 295, 0.0005, 0.005),
+    ("binance", True, 5, 295, 0.0005, round(0.0031249999999999997, 8)),
+    ("binance", False, 1, 295, 0.0005, 0.0),
+    ("binance", True, 1, 295, 0.0005, 0.003125),
+
+    ("kraken", False, 3, 10, 0.0005, 0.040),
+    ("kraken", True, 3, 10, 0.0005, 0.030),
+    ("kraken", False, 3, 295, 0.0005, 0.06),
+    ("kraken", True, 3, 295, 0.0005, 0.045),
+    ("kraken", False, 3, 295, 0.00025, 0.03),
+    ("kraken", True, 3, 295, 0.00025, 0.0225),
+    ("kraken", False, 5, 295, 0.0005, round(0.07200000000000001, 8)),
+    ("kraken", True, 5, 295, 0.0005, 0.045),
+    ("kraken", False, 1, 295, 0.0005, 0.0),
+    ("kraken", True, 1, 295, 0.0005, 0.045),
+
+])
 @pytest.mark.usefixtures("init_persistence")
-def test_interest(market_buy_order_usdt, fee):
+def test_interest(market_buy_order_usdt, fee, exchange, is_short, lev, minutes, rate, interest):
     """
         10min, 5hr limit trade on Binance/Kraken at 3x,5x leverage
         fee: 0.25 % quote
@@ -230,114 +252,27 @@ def test_interest(market_buy_order_usdt, fee):
         stake_amount=20.0,
         amount=30.0,
         open_rate=2.0,
-        open_date=datetime.utcnow() - timedelta(hours=0, minutes=10),
+        open_date=datetime.utcnow() - timedelta(minutes=minutes),
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='binance',
-        leverage=3.0,
-        interest_rate=0.0005,
+        exchange=exchange,
+        leverage=lev,
+        interest_rate=rate,
+        is_short=is_short
     )
 
-    # 10min, 3x leverage
-    # binance
-    assert round(float(trade.calculate_interest()), 8) == round(0.0008333333333333334, 8)
-    # kraken
-    trade.exchange = "kraken"
-    assert float(trade.calculate_interest()) == 0.040
-    # Short
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    # binace
-    trade.exchange = "binance"
-    assert float(trade.calculate_interest()) == 0.000625
-    # kraken
-    trade.exchange = "kraken"
-    assert isclose(float(trade.calculate_interest()), 0.030)
-
-    # 5hr, long
-    trade.open_date = datetime.utcnow() - timedelta(hours=4, minutes=55)
-    trade.is_short = False
-    trade.recalc_open_trade_value()
-    # binance
-    trade.exchange = "binance"
-    assert round(float(trade.calculate_interest()), 8) == round(0.004166666666666667, 8)
-    # kraken
-    trade.exchange = "kraken"
-    assert float(trade.calculate_interest()) == 0.06
-    # short
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    # binace
-    trade.exchange = "binance"
-    assert round(float(trade.calculate_interest()), 8) == round(0.0031249999999999997, 8)
-    # kraken
-    trade.exchange = "kraken"
-    assert float(trade.calculate_interest()) == 0.045
-
-    # 0.00025 interest, 5hr, long
-    trade.is_short = False
-    trade.recalc_open_trade_value()
-    # binance
-    trade.exchange = "binance"
-    assert round(float(trade.calculate_interest(interest_rate=0.00025)),
-                 8) == round(0.0020833333333333333, 8)
-    # kraken
-    trade.exchange = "kraken"
-    assert isclose(float(trade.calculate_interest(interest_rate=0.00025)), 0.03)
-    # short
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    # binace
-    trade.exchange = "binance"
-    assert round(float(trade.calculate_interest(interest_rate=0.00025)),
-                 8) == round(0.0015624999999999999, 8)
-    # kraken
-    trade.exchange = "kraken"
-    assert float(trade.calculate_interest(interest_rate=0.00025)) == 0.0225
-
-    # 5x leverage, 0.0005 interest, 5hr, long
-    trade.is_short = False
-    trade.recalc_open_trade_value()
-    trade.leverage = 5.0
-    # binance
-    trade.exchange = "binance"
-    assert round(float(trade.calculate_interest()), 8) == 0.005
-    # kraken
-    trade.exchange = "kraken"
-    assert float(trade.calculate_interest()) == round(0.07200000000000001, 8)
-    # short
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    # binace
-    trade.exchange = "binance"
-    assert round(float(trade.calculate_interest()), 8) == round(0.0031249999999999997, 8)
-    # kraken
-    trade.exchange = "kraken"
-    assert float(trade.calculate_interest()) == 0.045
-
-    # 1x leverage, 0.0005 interest, 5hr
-    trade.is_short = False
-    trade.recalc_open_trade_value()
-    trade.leverage = 1.0
-    # binance
-    trade.exchange = "binance"
-    assert float(trade.calculate_interest()) == 0.0
-    # kraken
-    trade.exchange = "kraken"
-    assert float(trade.calculate_interest()) == 0.0
-    # short
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    # binace
-    trade.exchange = "binance"
-    assert float(trade.calculate_interest()) == 0.003125
-    # kraken
-    trade.exchange = "kraken"
-    assert float(trade.calculate_interest()) == 0.045
+    assert round(float(trade.calculate_interest()), 8) == interest
 
 
+@pytest.mark.parametrize('is_short,lev,borrowed', [
+    (False, 1.0, 0.0),
+    (True, 1.0, 30.0),
+    (False, 3.0, 40.0),
+    (True, 3.0, 30.0),
+])
 @pytest.mark.usefixtures("init_persistence")
-def test_borrowed(limit_buy_order_usdt, limit_sell_order_usdt, fee, caplog):
+def test_borrowed(limit_buy_order_usdt, limit_sell_order_usdt, fee,
+                  caplog, is_short, lev, borrowed):
     """
         10 minute limit trade on Binance/Kraken at 1x, 3x leverage
         fee: 0.25% quote
@@ -411,20 +346,19 @@ def test_borrowed(limit_buy_order_usdt, limit_sell_order_usdt, fee, caplog):
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         exchange='binance',
+        is_short=is_short,
+        leverage=lev
     )
-    assert trade.borrowed == 0
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    assert trade.borrowed == 30.0
-    trade.leverage = 3.0
-    assert trade.borrowed == 30.0
-    trade.is_short = False
-    trade.recalc_open_trade_value()
-    assert trade.borrowed == 40.0
+    assert trade.borrowed == borrowed
 
 
+@pytest.mark.parametrize('is_short,open_rate,close_rate,lev,profit', [
+    (False, 2.0, 2.2, 1.0, round(0.0945137157107232, 8)),
+    (True, 2.2, 2.0, 3.0, round(0.2589996297562085, 8))
+])
 @pytest.mark.usefixtures("init_persistence")
-def test_update_limit_order(limit_buy_order_usdt, limit_sell_order_usdt, fee, caplog):
+def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_usdt,
+                            is_short, open_rate, close_rate, lev, profit):
     """
         10 minute limit trade on Binance/Kraken at 1x, 3x leverage
         fee: 0.25% quote
@@ -494,84 +428,52 @@ def test_update_limit_order(limit_buy_order_usdt, limit_sell_order_usdt, fee, ca
 
     """
 
+    enter_order = limit_sell_order_usdt if is_short else limit_buy_order_usdt
+    exit_order = limit_buy_order_usdt if is_short else limit_sell_order_usdt
+    enter_side, exit_side = get_sides(is_short)
+
     trade = Trade(
         id=2,
         pair='ADA/USDT',
         stake_amount=60.0,
-        open_rate=2.0,
-        amount=30.0,
-        is_open=True,
-        open_date=arrow.utcnow().datetime,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        exchange='binance'
-    )
-    assert trade.open_order_id is None
-    assert trade.close_profit is None
-    assert trade.close_date is None
-
-    trade.open_order_id = 'something'
-    trade.update(limit_buy_order_usdt)
-    assert trade.open_order_id is None
-    assert trade.open_rate == 2.00
-    assert trade.close_profit is None
-    assert trade.close_date is None
-    assert log_has_re(r"LIMIT_BUY has been fulfilled for Trade\(id=2, "
-                      r'pair=ADA/USDT, amount=30.00000000, '
-                      r"is_short=False, leverage=1.0, open_rate=2.00000000, open_since=.*\).",
-                      caplog)
-
-    caplog.clear()
-    trade.open_order_id = 'something'
-    trade.update(limit_sell_order_usdt)
-    assert trade.open_order_id is None
-    assert trade.close_rate == 2.20
-    assert trade.close_profit == round(0.0945137157107232, 8)
-    assert trade.close_date is not None
-    assert log_has_re(r"LIMIT_SELL has been fulfilled for Trade\(id=2, "
-                      r"pair=ADA/USDT, amount=30.00000000, "
-                      r"is_short=False, leverage=1.0, open_rate=2.00000000, open_since=.*\).",
-                      caplog)
-    caplog.clear()
-
-    trade = Trade(
-        id=226531,
-        pair='ADA/USDT',
-        stake_amount=20.0,
-        open_rate=2.0,
+        open_rate=open_rate,
         amount=30.0,
         is_open=True,
         open_date=arrow.utcnow().datetime,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         exchange='binance',
-        is_short=True,
-        leverage=3.0,
+        is_short=is_short,
         interest_rate=0.0005,
+        leverage=lev
     )
-    trade.open_order_id = 'something'
-    trade.update(limit_sell_order_usdt)
-
     assert trade.open_order_id is None
-    assert trade.open_rate == 2.20
     assert trade.close_profit is None
     assert trade.close_date is None
 
-    assert log_has_re(r"LIMIT_SELL has been fulfilled for Trade\(id=226531, "
-                      r"pair=ADA/USDT, amount=30.00000000, "
-                      r"is_short=True, leverage=3.0, open_rate=2.20000000, open_since=.*\).",
-                      caplog)
-    caplog.clear()
-
     trade.open_order_id = 'something'
-    trade.update(limit_buy_order_usdt)
+    trade.update(enter_order)
     assert trade.open_order_id is None
-    assert trade.close_rate == 2.00
-    assert trade.close_profit == round(0.2589996297562085, 8)
+    assert trade.open_rate == open_rate
+    assert trade.close_profit is None
+    assert trade.close_date is None
+    assert log_has_re(f"LIMIT_{enter_side.upper()} has been fulfilled for "
+                      r"Trade\(id=2, pair=ADA/USDT, amount=30.00000000, "
+                      f"is_short={is_short}, leverage={lev}, open_rate={open_rate}0000000, "
+                      r"open_since=.*\).",
+                      caplog)
+
+    caplog.clear()
+    trade.open_order_id = 'something'
+    trade.update(exit_order)
+    assert trade.open_order_id is None
+    assert trade.close_rate == close_rate
+    assert trade.close_profit == profit
     assert trade.close_date is not None
-    assert log_has_re(r"LIMIT_BUY has been fulfilled for Trade\(id=226531, "
-                      r"pair=ADA/USDT, amount=30.00000000, "
-                      r"is_short=True, leverage=3.0, open_rate=2.20000000, open_since=.*\).",
+    assert log_has_re(f"LIMIT_{exit_side.upper()} has been fulfilled for "
+                      r"Trade\(id=2, pair=ADA/USDT, amount=30.00000000, "
+                      f"is_short={is_short}, leverage={lev}, open_rate={open_rate}0000000, "
+                      r"open_since=.*\).",
                       caplog)
     caplog.clear()
 
@@ -616,9 +518,21 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee,
                       caplog)
 
 
+@pytest.mark.parametrize('exchange,is_short,lev,open_value,close_value,profit,profit_ratio', [
+    ("binance", False, 1, 60.15, 65.835, 5.685, 0.0945137157107232),
+    ("binance", True, 1, 59.850, 66.1663784375, -6.316378437500013, -0.1055368159983292),
+    ("binance", False, 3, 60.15, 65.83416667, 5.684166670000003, 0.2834995845386534),
+    ("binance", True, 3, 59.85, 66.1663784375, -6.316378437500013, -0.3166104479949876),
+
+    ("kraken", False, 1, 60.15, 65.835, 5.685, 0.0945137157107232),
+    ("kraken", True, 1, 59.850, 66.231165, -6.381165, -0.106619298245614),
+    ("kraken", False, 3, 60.15, 65.795, 5.645, 0.2815461346633419),
+    ("kraken", True, 3, 59.850, 66.231165, -6.381165000000003, -0.319857894736842),
+])
 @pytest.mark.usefixtures("init_persistence")
-def test_calc_open_close_trade_price(limit_buy_order_usdt, limit_sell_order_usdt, fee):
-    trade = Trade(
+def test_calc_open_close_trade_price(limit_buy_order_usdt, limit_sell_order_usdt, fee, exchange,
+                                     is_short, lev, open_value, close_value, profit, profit_ratio):
+    trade: Trade = Trade(
         pair='ADA/USDT',
         stake_amount=60.0,
         open_rate=2.0,
@@ -627,58 +541,25 @@ def test_calc_open_close_trade_price(limit_buy_order_usdt, limit_sell_order_usdt
         interest_rate=0.0005,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='binance',
+        exchange=exchange,
+        is_short=is_short,
+        leverage=lev
     )
 
-    trade.open_order_id = 'something'
+    trade.open_order_id = f'something-{is_short}-{lev}-{exchange}'
+
     trade.update(limit_buy_order_usdt)
     trade.update(limit_sell_order_usdt)
-    # 1x leverage, binance
-    assert trade._calc_open_trade_value() == 60.15
-    assert isclose(trade.calc_close_trade_value(), 65.835)
-    assert trade.calc_profit() == 5.685
-    assert trade.calc_profit_ratio() == round(0.0945137157107232, 8)
-    # 3x leverage, binance
-    trade.leverage = 3
-    trade.exchange = "binance"
-    assert trade._calc_open_trade_value() == 60.15
-    assert round(trade.calc_close_trade_value(), 8) == 65.83416667
-    assert trade.calc_profit() == round(5.684166670000003, 8)
-    assert trade.calc_profit_ratio() == round(0.2834995845386534, 8)
-    trade.exchange = "kraken"
-    # 3x leverage, kraken
-    assert trade._calc_open_trade_value() == 60.15
-    assert trade.calc_close_trade_value() == 65.795
-    assert trade.calc_profit() == 5.645
-    assert trade.calc_profit_ratio() == round(0.2815461346633419, 8)
-    trade.is_short = True
+    trade.open_rate = 2.0
+    trade.close_rate = 2.2
     trade.recalc_open_trade_value()
-    # 3x leverage, short, kraken
-    assert trade._calc_open_trade_value() == 59.850
-    assert trade.calc_close_trade_value() == 66.231165
-    assert trade.calc_profit() == round(-6.381165000000003, 8)
-    assert trade.calc_profit_ratio() == round(-0.319857894736842, 8)
-    trade.exchange = "binance"
-    # 3x leverage, short, binance
-    assert trade._calc_open_trade_value() == 59.85
-    assert trade.calc_close_trade_value() == 66.1663784375
-    assert trade.calc_profit() == round(-6.316378437500013, 8)
-    assert trade.calc_profit_ratio() == round(-0.3166104479949876, 8)
-    # 1x leverage, short, binance
-    trade.leverage = 1.0
-    assert trade._calc_open_trade_value() == 59.850
-    assert trade.calc_close_trade_value() == 66.1663784375
-    assert trade.calc_profit() == round(-6.316378437500013, 8)
-    assert trade.calc_profit_ratio() == round(-0.1055368159983292, 8)
-    # 1x leverage, short, kraken
-    trade.exchange = "kraken"
-    assert trade._calc_open_trade_value() == 59.850
-    assert trade.calc_close_trade_value() == 66.231165
-    assert trade.calc_profit() == -6.381165
-    assert trade.calc_profit_ratio() == round(-0.106619298245614, 8)
+    assert isclose(trade._calc_open_trade_value(), open_value)
+    assert isclose(trade.calc_close_trade_value(), close_value)
+    assert isclose(trade.calc_profit(), round(profit, 8))
+    assert isclose(trade.calc_profit_ratio(), round(profit_ratio, 8))
 
 
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_trade_close(limit_buy_order_usdt, limit_sell_order_usdt, fee):
     trade = Trade(
         pair='ADA/USDT',
@@ -709,7 +590,7 @@ def test_trade_close(limit_buy_order_usdt, limit_sell_order_usdt, fee):
     assert trade.close_date == new_date
 
 
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_calc_close_trade_price_exception(limit_buy_order_usdt, fee):
     trade = Trade(
         pair='ADA/USDT',
@@ -726,7 +607,7 @@ def test_calc_close_trade_price_exception(limit_buy_order_usdt, fee):
     assert trade.calc_close_trade_value() == 0.0
 
 
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_update_open_order(limit_buy_order_usdt):
     trade = Trade(
         pair='ADA/USDT',
@@ -750,7 +631,7 @@ def test_update_open_order(limit_buy_order_usdt):
     assert trade.close_date is None
 
 
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_update_invalid_order(limit_buy_order_usdt):
     trade = Trade(
         pair='ADA/USDT',
@@ -766,8 +647,27 @@ def test_update_invalid_order(limit_buy_order_usdt):
         trade.update(limit_buy_order_usdt)
 
 
+@pytest.mark.parametrize('exchange', ['binance', 'kraken'])
+@pytest.mark.parametrize('lev', [1, 3])
+@pytest.mark.parametrize('is_short,fee_rate,result', [
+    (False, 0.003, 60.18),
+    (False, 0.0025, 60.15),
+    (False, 0.003, 60.18),
+    (False, 0.0025, 60.15),
+    (True, 0.003, 59.82),
+    (True, 0.0025, 59.85),
+    (True, 0.003, 59.82),
+    (True, 0.0025, 59.85)
+])
 @pytest.mark.usefixtures("init_persistence")
-def test_calc_open_trade_value(limit_buy_order_usdt, fee):
+def test_calc_open_trade_value(
+    limit_buy_order_usdt,
+    exchange,
+    lev,
+    is_short,
+    fee_rate,
+    result
+):
     # 10 minute limit trade on Binance/Kraken at 1x, 3x leverage
     # fee: 0.25 %, 0.3% quote
     # open_rate: 2.00 quote
@@ -787,90 +687,104 @@ def test_calc_open_trade_value(limit_buy_order_usdt, fee):
         stake_amount=60.0,
         amount=30.0,
         open_rate=2.0,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        exchange='binance',
+        open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=10),
+        fee_open=fee_rate,
+        fee_close=fee_rate,
+        exchange=exchange,
+        leverage=lev,
+        is_short=is_short
     )
     trade.open_order_id = 'open_trade'
-    trade.update(limit_buy_order_usdt)
 
     # Get the open rate price with the standard fee rate
-    assert trade._calc_open_trade_value() == 60.15
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    assert trade._calc_open_trade_value() == 59.85
-    trade.leverage = 3
-    trade.exchange = "binance"
-    assert trade._calc_open_trade_value() == 59.85
-    trade.is_short = False
-    trade.recalc_open_trade_value()
-    assert trade._calc_open_trade_value() == 60.15
-
-    # Get the open rate price with a custom fee rate
-    trade.fee_open = 0.003
-
-    assert trade._calc_open_trade_value() == 60.18
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    assert trade._calc_open_trade_value() == 59.82
+    assert trade._calc_open_trade_value() == result
 
 
+@pytest.mark.parametrize('exchange,is_short,lev,open_rate,close_rate,fee_rate,result', [
+    ('binance', False, 1, 2.0, 2.5, 0.0025, 74.8125),
+    ('binance', False, 1, 2.0, 2.5, 0.003, 74.775),
+    ('binance', False, 1, 2.0, 2.2, 0.005, 65.67),
+    ('binance', False, 3, 2.0, 2.5, 0.0025, 74.81166667),
+    ('binance', False, 3, 2.0, 2.5, 0.003, 74.77416667),
+    ('kraken', False, 3, 2.0, 2.5, 0.0025, 74.7725),
+    ('kraken', False, 3, 2.0, 2.5, 0.003, 74.735),
+    ('kraken', True, 3, 2.2, 2.5, 0.0025, 75.2626875),
+    ('kraken', True, 3, 2.2, 2.5, 0.003, 75.300225),
+    ('binance', True, 3, 2.2, 2.5, 0.0025, 75.18906641),
+    ('binance', True, 3, 2.2, 2.5, 0.003, 75.22656719),
+    ('binance', True, 1, 2.2, 2.5, 0.0025, 75.18906641),
+    ('binance', True, 1, 2.2, 2.5, 0.003, 75.22656719),
+    ('kraken', True, 1, 2.2, 2.5, 0.0025, 75.2626875),
+    ('kraken', True, 1, 2.2, 2.5, 0.003, 75.300225),
+])
 @pytest.mark.usefixtures("init_persistence")
-def test_calc_close_trade_price(limit_buy_order_usdt, limit_sell_order_usdt, fee):
+def test_calc_close_trade_price(limit_buy_order_usdt, limit_sell_order_usdt, open_rate,
+                                exchange, is_short, lev, close_rate, fee_rate, result):
     trade = Trade(
         pair='ADA/USDT',
         stake_amount=60.0,
         amount=30.0,
-        open_rate=2.0,
+        open_rate=open_rate,
         open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=10),
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        exchange='binance',
+        fee_open=fee_rate,
+        fee_close=fee_rate,
+        exchange=exchange,
         interest_rate=0.0005,
+        is_short=is_short,
+        leverage=lev
     )
     trade.open_order_id = 'close_trade'
-    trade.update(limit_buy_order_usdt)
-
-    # 1x leverage binance
-    assert trade.calc_close_trade_value(rate=2.5) == 74.8125
-    assert trade.calc_close_trade_value(rate=2.5, fee=0.003) == 74.775
-    trade.update(limit_sell_order_usdt)
-    assert trade.calc_close_trade_value(fee=0.005) == 65.67
-
-    # 3x leverage binance
-    trade.leverage = 3.0
-    assert round(trade.calc_close_trade_value(rate=2.5), 8) == 74.81166667
-    assert round(trade.calc_close_trade_value(rate=2.5, fee=0.003), 8) == 74.77416667
-
-    # 3x leverage kraken
-    trade.exchange = "kraken"
-    assert trade.calc_close_trade_value(rate=2.5) == 74.7725
-    assert trade.calc_close_trade_value(rate=2.5, fee=0.003) == 74.735
-
-    # 3x leverage kraken, short
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    assert round(trade.calc_close_trade_value(rate=2.5), 8) == 75.2626875
-    assert trade.calc_close_trade_value(rate=2.5, fee=0.003) == 75.300225
-
-    # 3x leverage binance, short
-    trade.exchange = "binance"
-    assert round(trade.calc_close_trade_value(rate=2.5), 8) == 75.18906641
-    assert round(trade.calc_close_trade_value(rate=2.5, fee=0.003), 8) == 75.22656719
-
-    trade.leverage = 1.0
-    # 1x leverage binance, short
-    assert round(trade.calc_close_trade_value(rate=2.5), 8) == 75.18906641
-    assert round(trade.calc_close_trade_value(rate=2.5, fee=0.003), 8) == 75.22656719
-
-    # 1x leverage kraken, short
-    trade.exchange = "kraken"
-    assert round(trade.calc_close_trade_value(rate=2.5), 8) == 75.2626875
-    assert trade.calc_close_trade_value(rate=2.5, fee=0.003) == 75.300225
+    assert round(trade.calc_close_trade_value(rate=close_rate, fee=fee_rate), 8) == result
 
 
+@pytest.mark.parametrize('exchange,is_short,lev,close_rate,fee_close,profit,profit_ratio', [
+    ('binance', False, 1, 2.1, 0.0025, 2.6925, 0.04476309226932673),
+    ('binance', False, 3, 2.1, 0.0025, 2.69166667, 0.13424771421446402),
+    ('binance', True, 1, 2.1, 0.0025, -3.308815781249997, -0.05528514254385963),
+    ('binance', True, 3, 2.1, 0.0025, -3.308815781249997, -0.1658554276315789),
+
+    ('binance', False, 1, 1.9, 0.0025, -3.2925, -0.05473815461346632),
+    ('binance', False, 3, 1.9, 0.0025, -3.29333333, -0.16425602643391513),
+    ('binance', True, 1, 1.9, 0.0025, 2.7063095312499996, 0.045218204365079395),
+    ('binance', True, 3, 1.9, 0.0025, 2.7063095312499996, 0.13565461309523819),
+
+    ('binance', False, 1, 2.2, 0.0025, 5.685, 0.0945137157107232),
+    ('binance', False, 3, 2.2, 0.0025, 5.68416667, 0.2834995845386534),
+    ('binance', True, 1, 2.2, 0.0025, -6.316378437499999, -0.1055368159983292),
+    ('binance', True, 3, 2.2, 0.0025, -6.316378437499999, -0.3166104479949876),
+
+    ('kraken', False, 1, 2.1, 0.0025, 2.6925, 0.04476309226932673),
+    ('kraken', False, 3, 2.1, 0.0025, 2.6525, 0.13229426433915248),
+    ('kraken', True, 1, 2.1, 0.0025, -3.3706575, -0.05631842105263152),
+    ('kraken', True, 3, 2.1, 0.0025, -3.3706575, -0.16895526315789455),
+
+    ('kraken', False, 1, 1.9, 0.0025, -3.2925, -0.05473815461346632),
+    ('kraken', False, 3, 1.9, 0.0025, -3.3325, -0.16620947630922667),
+    ('kraken', True, 1, 1.9, 0.0025, 2.6503575, 0.04428333333333334),
+    ('kraken', True, 3, 1.9, 0.0025, 2.6503575, 0.13285000000000002),
+
+    ('kraken', False, 1, 2.2, 0.0025, 5.685, 0.0945137157107232),
+    ('kraken', False, 3, 2.2, 0.0025, 5.645, 0.2815461346633419),
+    ('kraken', True, 1, 2.2, 0.0025, -6.381165, -0.106619298245614),
+    ('kraken', True, 3, 2.2, 0.0025, -6.381165, -0.319857894736842),
+
+    ('binance', False, 1, 2.1, 0.003, 2.6610000000000014, 0.04423940149625927),
+    ('binance', False, 1, 1.9, 0.003, -3.320999999999998, -0.05521197007481293),
+    ('binance', False, 1, 2.2, 0.003, 5.652000000000008, 0.09396508728179565),
+])
 @pytest.mark.usefixtures("init_persistence")
-def test_calc_profit(limit_buy_order_usdt, limit_sell_order_usdt, fee):
+def test_calc_profit(
+    limit_buy_order_usdt,
+    limit_sell_order_usdt,
+    fee,
+    exchange,
+    is_short,
+    lev,
+    close_rate,
+    fee_close,
+    profit,
+    profit_ratio
+):
     """
         10 minute limit trade on Binance/Kraken at 1x, 3x leverage
         arguments:
@@ -1007,201 +921,19 @@ def test_calc_profit(limit_buy_order_usdt, limit_sell_order_usdt, fee):
         open_rate=2.0,
         open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=10),
         interest_rate=0.0005,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        exchange='binance'
+        exchange=exchange,
+        is_short=is_short,
+        leverage=lev,
+        fee_open=0.0025,
+        fee_close=fee_close
     )
     trade.open_order_id = 'something'
-    trade.update(limit_buy_order_usdt)  # Buy @ 2.0
 
-    # 1x Leverage, long
-    # Custom closing rate and regular fee rate
-    # Higher than open rate - 2.1 quote
-    assert trade.calc_profit(rate=2.1) == 2.6925
-    # Lower than open rate - 1.9 quote
-    assert trade.calc_profit(rate=1.9) == round(-3.292499999999997, 8)
-
-    # fee 0.003
-    # Higher than open rate - 2.1 quote
-    assert trade.calc_profit(rate=2.1, fee=0.003) == 2.661
-    # Lower than open rate - 1.9 quote
-    assert trade.calc_profit(rate=1.9, fee=0.003) == round(-3.320999999999998, 8)
-
-    # Test when we apply a Sell order. Sell higher than open rate @ 2.2
-    trade.update(limit_sell_order_usdt)
-    assert trade.calc_profit() == round(5.684999999999995, 8)
-
-    # Test with a custom fee rate on the close trade
-    assert trade.calc_profit(fee=0.003) == round(5.652000000000008, 8)
-
-    trade.open_trade_value = 0.0
-    trade.open_trade_value = trade._calc_open_trade_value()
-
-    # 3x leverage, long ###################################################
-    trade.leverage = 3.0
-    # Higher than open rate - 2.1 quote
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit(rate=2.1, fee=0.0025) == 2.69166667
-    trade.exchange = "kraken"
-    assert trade.calc_profit(rate=2.1, fee=0.0025) == 2.6525
-
-    # 1.9 quote
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit(rate=1.9, fee=0.0025) == -3.29333333
-    trade.exchange = "kraken"
-    assert trade.calc_profit(rate=1.9, fee=0.0025) == -3.3325
-
-    # 2.2 quote
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit(fee=0.0025) == 5.68416667
-    trade.exchange = "kraken"
-    assert trade.calc_profit(fee=0.0025) == 5.645
-
-    # 3x leverage, short ###################################################
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    # 2.1 quote - Higher than open rate
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit(rate=2.1, fee=0.0025) == round(-3.308815781249997, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit(rate=2.1, fee=0.0025) == -3.3706575
-
-    # 1.9 quote - Lower than open rate
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit(rate=1.9, fee=0.0025) == round(2.7063095312499996, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit(rate=1.9, fee=0.0025) == 2.6503575
-
-    # Test when we apply a Sell order. Uses sell order used above
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit(fee=0.0025) == round(-6.316378437499999, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit(fee=0.0025) == -6.381165
-
-    # 1x leverage, short ###################################################
-    trade.leverage = 1.0
-    # 2.1 quote - Higher than open rate
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit(rate=2.1, fee=0.0025) == round(-3.308815781249997, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit(rate=2.1, fee=0.0025) == -3.3706575
-
-    # 1.9 quote - Lower than open rate
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit(rate=1.9, fee=0.0025) == round(2.7063095312499996, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit(rate=1.9, fee=0.0025) == 2.6503575
-
-    # Test when we apply a Sell order. Uses sell order used above
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit(fee=0.0025) == round(-6.316378437499999, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit(fee=0.0025) == -6.381165
+    assert trade.calc_profit(rate=close_rate) == round(profit, 8)
+    assert trade.calc_profit_ratio(rate=close_rate) == round(profit_ratio, 8)
 
 
-@pytest.mark.usefixtures("init_persistence")
-def test_calc_profit_ratio(limit_buy_order_usdt, limit_sell_order_usdt, fee):
-    trade = Trade(
-        pair='ADA/USDT',
-        stake_amount=60.0,
-        amount=30.0,
-        open_rate=2.0,
-        open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=10),
-        interest_rate=0.0005,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        exchange='binance'
-    )
-    trade.open_order_id = 'something'
-    trade.update(limit_buy_order_usdt)  # Buy @ 2.0
-
-    # 1x Leverage, long
-    # Custom closing rate and regular fee rate
-    # Higher than open rate - 2.1 quote
-    assert trade.calc_profit_ratio(rate=2.1) == round(0.04476309226932673, 8)
-    # Lower than open rate - 1.9 quote
-    assert trade.calc_profit_ratio(rate=1.9) == round(-0.05473815461346632, 8)
-
-    # fee 0.003
-    # Higher than open rate - 2.1 quote
-    assert trade.calc_profit_ratio(rate=2.1, fee=0.003) == round(0.04423940149625927, 8)
-    # Lower than open rate - 1.9 quote
-    assert trade.calc_profit_ratio(rate=1.9, fee=0.003) == round(-0.05521197007481293, 8)
-
-    # Test when we apply a Sell order. Sell higher than open rate @ 2.2
-    trade.update(limit_sell_order_usdt)
-    assert trade.calc_profit_ratio() == round(0.0945137157107232, 8)
-
-    # Test with a custom fee rate on the close trade
-    assert trade.calc_profit_ratio(fee=0.003) == round(0.09396508728179565, 8)
-
-    trade.open_trade_value = 0.0
-    assert trade.calc_profit_ratio(fee=0.003) == 0.0
-    trade.open_trade_value = trade._calc_open_trade_value()
-
-    # 3x leverage, long ###################################################
-    trade.leverage = 3.0
-    # 2.1 quote - Higher than open rate
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit_ratio(rate=2.1) == round(0.13424771421446402, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit_ratio(rate=2.1) == round(0.13229426433915248, 8)
-
-    # 1.9 quote - Lower than open rate
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit_ratio(rate=1.9) == round(-0.16425602643391513, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit_ratio(rate=1.9) == round(-0.16620947630922667, 8)
-
-    # Test when we apply a Sell order. Uses sell order used above
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit_ratio() == round(0.2834995845386534, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit_ratio() == round(0.2815461346633419, 8)
-
-    # 3x leverage, short ###################################################
-    trade.is_short = True
-    trade.recalc_open_trade_value()
-    # 2.1 quote - Higher than open rate
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit_ratio(rate=2.1) == round(-0.1658554276315789, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit_ratio(rate=2.1) == round(-0.16895526315789455, 8)
-
-    # 1.9 quote - Lower than open rate
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit_ratio(rate=1.9) == round(0.13565461309523819, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit_ratio(rate=1.9) == round(0.13285000000000002, 8)
-
-    # Test when we apply a Sell order. Uses sell order used above
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit_ratio() == round(-0.3166104479949876, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit_ratio() == round(-0.319857894736842, 8)
-
-    # 1x leverage, short ###################################################
-    trade.leverage = 1.0
-    # 2.1 quote - Higher than open rate
-    trade.exchange = "binance"  # binance
-    assert trade.calc_profit_ratio(rate=2.1) == round(-0.05528514254385963, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit_ratio(rate=2.1) == round(-0.05631842105263152, 8)
-
-    # 1.9 quote - Lower than open rate
-    trade.exchange = "binance"
-    assert trade.calc_profit_ratio(rate=1.9) == round(0.045218204365079395, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit_ratio(rate=1.9) == round(0.04428333333333334, 8)
-
-    # Test when we apply a Sell order. Uses sell order used above
-    trade.exchange = "binance"
-    assert trade.calc_profit_ratio() == round(-0.1055368159983292, 8)
-    trade.exchange = "kraken"
-    assert trade.calc_profit_ratio() == round(-0.106619298245614, 8)
-
-
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_clean_dry_run_db(default_conf, fee):
 
     # Simulate dry_run entries
@@ -1612,8 +1344,8 @@ def test_adjust_min_max_rates(fee):
     assert trade.min_rate == 0.91
 
 
-@pytest.mark.usefixtures("init_persistence")
-@pytest.mark.parametrize('use_db', [True, False])
+@ pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.parametrize('use_db', [True, False])
 def test_get_open(fee, use_db):
     Trade.use_db = use_db
     Trade.reset_trades()
@@ -1624,8 +1356,8 @@ def test_get_open(fee, use_db):
     Trade.use_db = True
 
 
-@pytest.mark.usefixtures("init_persistence")
-@pytest.mark.parametrize('use_db', [True, False])
+@ pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.parametrize('use_db', [True, False])
 def test_get_open_lev(fee, use_db):
     Trade.use_db = use_db
     Trade.reset_trades()
@@ -1636,7 +1368,7 @@ def test_get_open_lev(fee, use_db):
     Trade.use_db = True
 
 
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_to_json(default_conf, fee):
 
     # Simulate dry_run entries
@@ -1969,8 +1701,8 @@ def test_fee_updated(fee):
     assert not trade.fee_updated('asfd')
 
 
-@pytest.mark.usefixtures("init_persistence")
-@pytest.mark.parametrize('use_db', [True, False])
+@ pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.parametrize('use_db', [True, False])
 def test_total_open_trades_stakes(fee, use_db):
 
     Trade.use_db = use_db
@@ -1984,8 +1716,8 @@ def test_total_open_trades_stakes(fee, use_db):
     Trade.use_db = True
 
 
-@pytest.mark.usefixtures("init_persistence")
-@pytest.mark.parametrize('use_db', [True, False])
+@ pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.parametrize('use_db', [True, False])
 def test_get_total_closed_profit(fee, use_db):
 
     Trade.use_db = use_db
@@ -1999,8 +1731,8 @@ def test_get_total_closed_profit(fee, use_db):
     Trade.use_db = True
 
 
-@pytest.mark.usefixtures("init_persistence")
-@pytest.mark.parametrize('use_db', [True, False])
+@ pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.parametrize('use_db', [True, False])
 def test_get_trades_proxy(fee, use_db):
     Trade.use_db = use_db
     Trade.reset_trades()
@@ -2032,7 +1764,7 @@ def test_get_trades_backtest():
     Trade.use_db = True
 
 
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_get_overall_performance(fee):
 
     create_mock_trades(fee)
@@ -2044,7 +1776,7 @@ def test_get_overall_performance(fee):
     assert 'count' in res[0]
 
 
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_get_best_pair(fee):
 
     res = Trade.get_best_pair()
@@ -2057,7 +1789,7 @@ def test_get_best_pair(fee):
     assert res[1] == 0.01
 
 
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_get_best_pair_lev(fee):
 
     res = Trade.get_best_pair()
@@ -2070,7 +1802,7 @@ def test_get_best_pair_lev(fee):
     assert res[1] == 0.1713156134055116
 
 
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_update_order_from_ccxt(caplog):
     # Most basic order return (only has orderid)
     o = Order.parse_from_ccxt_object({'id': '1234'}, 'ADA/USDT', 'buy')
@@ -2131,7 +1863,7 @@ def test_update_order_from_ccxt(caplog):
     Order.update_orders([o], {'id': '1234'})
 
 
-@pytest.mark.usefixtures("init_persistence")
+@ pytest.mark.usefixtures("init_persistence")
 def test_select_order(fee):
     create_mock_trades(fee)
 

From 994c3c3a4c5f36c02d249f4c13466b284c4991af Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 16 Sep 2021 07:13:25 +0200
Subject: [PATCH 11/13] Add some errorhandling for custom estimator

---
 freqtrade/optimize/hyperopt.py  | 14 ++++++++++----
 tests/optimize/test_hyperopt.py |  4 ++++
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 56d11934a..9549b4054 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -45,7 +45,7 @@ progressbar.streams.wrap_stdout()
 logger = logging.getLogger(__name__)
 
 
-INITIAL_POINTS = 30
+INITIAL_POINTS = 5
 
 # Keep no more than SKOPT_MODEL_QUEUE_SIZE models
 # in the skopt model queue, to optimize memory consumption
@@ -366,13 +366,19 @@ class Hyperopt:
 
     def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
         estimator = self.custom_hyperopt.generate_estimator()
+
+        acq_optimizer = "sampling"
+        if isinstance(estimator, str):
+            if estimator not in ("GP", "RF", "ET", "GBRT"):
+                raise OperationalException(f"Estimator {estimator} not supported.")
+            else:
+                acq_optimizer = "auto"
+
         logger.info(f"Using estimator {estimator}.")
-        # TODO: Impact of changing acq_optimizer to "sampling" is unclear
-        # (other than that it fails with other optimizers when using custom sklearn regressors)
         return Optimizer(
             dimensions,
             base_estimator=estimator,
-            acq_optimizer="sampling",
+            acq_optimizer=acq_optimizer,
             n_initial_points=INITIAL_POINTS,
             acq_optimizer_kwargs={'n_jobs': cpu_count},
             random_state=self.random_state,
diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py
index b34c3a916..e4ce29d44 100644
--- a/tests/optimize/test_hyperopt.py
+++ b/tests/optimize/test_hyperopt.py
@@ -884,6 +884,10 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None:
     assert hyperopt.backtesting.strategy.buy_rsi.value != 35
     assert hyperopt.backtesting.strategy.sell_rsi.value != 74
 
+    hyperopt.custom_hyperopt.generate_estimator = lambda *args, **kwargs: 'ET1'
+    with pytest.raises(OperationalException, match="Estimator ET1 not supported."):
+        hyperopt.get_optimizer([], 2)
+
 
 def test_SKDecimal():
     space = SKDecimal(1, 2, decimals=2)

From dec2f377ff6e2bc815450703bc7d480871317c67 Mon Sep 17 00:00:00 2001
From: Sam Germain <sam.germain@usask.ca>
Date: Thu, 16 Sep 2021 16:25:02 -0600
Subject: [PATCH 12/13] Removed utils, moved get_sides to conftest

---
 freqtrade/utils/__init__.py  | 3 ---
 freqtrade/utils/get_sides.py | 5 -----
 tests/conftest.py            | 5 +++++
 tests/test_persistence.py    | 4 ++--
 4 files changed, 7 insertions(+), 10 deletions(-)
 delete mode 100644 freqtrade/utils/__init__.py
 delete mode 100644 freqtrade/utils/get_sides.py

diff --git a/freqtrade/utils/__init__.py b/freqtrade/utils/__init__.py
deleted file mode 100644
index 361a06c38..000000000
--- a/freqtrade/utils/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-# flake8: noqa: F401
-
-from freqtrade.utils.get_sides import get_sides
diff --git a/freqtrade/utils/get_sides.py b/freqtrade/utils/get_sides.py
deleted file mode 100644
index 9ab97e7b3..000000000
--- a/freqtrade/utils/get_sides.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from typing import Tuple
-
-
-def get_sides(is_short: bool) -> Tuple[str, str]:
-    return ("sell", "buy") if is_short else ("buy", "sell")
diff --git a/tests/conftest.py b/tests/conftest.py
index 188236f40..609823409 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -6,6 +6,7 @@ from copy import deepcopy
 from datetime import datetime, timedelta
 from functools import reduce
 from pathlib import Path
+from typing import Tuple
 from unittest.mock import MagicMock, Mock, PropertyMock
 
 import arrow
@@ -262,6 +263,10 @@ def create_mock_trades_with_leverage(fee, use_db: bool = True):
         Trade.query.session.flush()
 
 
+def get_sides(is_short: bool) -> Tuple[str, str]:
+    return ("sell", "buy") if is_short else ("buy", "sell")
+
+
 @pytest.fixture(autouse=True)
 def patch_coingekko(mocker) -> None:
     """
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 800e3f541..dbb1133c3 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -13,8 +13,8 @@ from sqlalchemy import create_engine, inspect, text
 from freqtrade import constants
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.persistence import LocalTrade, Order, Trade, clean_dry_run_db, init_db
-from freqtrade.utils import get_sides
-from tests.conftest import create_mock_trades, create_mock_trades_with_leverage, log_has, log_has_re
+from tests.conftest import (create_mock_trades, create_mock_trades_with_leverage, get_sides,
+                            log_has, log_has_re)
 
 
 def test_init_create_session(default_conf):

From 0ced05890adbbbd1e3afef1f2dcc26ea6c6c1515 Mon Sep 17 00:00:00 2001
From: Sam Germain <sam.germain@usask.ca>
Date: Thu, 16 Sep 2021 16:26:31 -0600
Subject: [PATCH 13/13] removed space between @ and pytest

---
 tests/test_persistence.py | 42 +++++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index dbb1133c3..acdd79350 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -559,7 +559,7 @@ def test_calc_open_close_trade_price(limit_buy_order_usdt, limit_sell_order_usdt
     assert isclose(trade.calc_profit_ratio(), round(profit_ratio, 8))
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_trade_close(limit_buy_order_usdt, limit_sell_order_usdt, fee):
     trade = Trade(
         pair='ADA/USDT',
@@ -590,7 +590,7 @@ def test_trade_close(limit_buy_order_usdt, limit_sell_order_usdt, fee):
     assert trade.close_date == new_date
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_calc_close_trade_price_exception(limit_buy_order_usdt, fee):
     trade = Trade(
         pair='ADA/USDT',
@@ -607,7 +607,7 @@ def test_calc_close_trade_price_exception(limit_buy_order_usdt, fee):
     assert trade.calc_close_trade_value() == 0.0
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_update_open_order(limit_buy_order_usdt):
     trade = Trade(
         pair='ADA/USDT',
@@ -631,7 +631,7 @@ def test_update_open_order(limit_buy_order_usdt):
     assert trade.close_date is None
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_update_invalid_order(limit_buy_order_usdt):
     trade = Trade(
         pair='ADA/USDT',
@@ -933,7 +933,7 @@ def test_calc_profit(
     assert trade.calc_profit_ratio(rate=close_rate) == round(profit_ratio, 8)
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_clean_dry_run_db(default_conf, fee):
 
     # Simulate dry_run entries
@@ -1344,8 +1344,8 @@ def test_adjust_min_max_rates(fee):
     assert trade.min_rate == 0.91
 
 
-@ pytest.mark.usefixtures("init_persistence")
-@ pytest.mark.parametrize('use_db', [True, False])
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize('use_db', [True, False])
 def test_get_open(fee, use_db):
     Trade.use_db = use_db
     Trade.reset_trades()
@@ -1356,8 +1356,8 @@ def test_get_open(fee, use_db):
     Trade.use_db = True
 
 
-@ pytest.mark.usefixtures("init_persistence")
-@ pytest.mark.parametrize('use_db', [True, False])
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize('use_db', [True, False])
 def test_get_open_lev(fee, use_db):
     Trade.use_db = use_db
     Trade.reset_trades()
@@ -1368,7 +1368,7 @@ def test_get_open_lev(fee, use_db):
     Trade.use_db = True
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_to_json(default_conf, fee):
 
     # Simulate dry_run entries
@@ -1701,8 +1701,8 @@ def test_fee_updated(fee):
     assert not trade.fee_updated('asfd')
 
 
-@ pytest.mark.usefixtures("init_persistence")
-@ pytest.mark.parametrize('use_db', [True, False])
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize('use_db', [True, False])
 def test_total_open_trades_stakes(fee, use_db):
 
     Trade.use_db = use_db
@@ -1716,8 +1716,8 @@ def test_total_open_trades_stakes(fee, use_db):
     Trade.use_db = True
 
 
-@ pytest.mark.usefixtures("init_persistence")
-@ pytest.mark.parametrize('use_db', [True, False])
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize('use_db', [True, False])
 def test_get_total_closed_profit(fee, use_db):
 
     Trade.use_db = use_db
@@ -1731,8 +1731,8 @@ def test_get_total_closed_profit(fee, use_db):
     Trade.use_db = True
 
 
-@ pytest.mark.usefixtures("init_persistence")
-@ pytest.mark.parametrize('use_db', [True, False])
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize('use_db', [True, False])
 def test_get_trades_proxy(fee, use_db):
     Trade.use_db = use_db
     Trade.reset_trades()
@@ -1764,7 +1764,7 @@ def test_get_trades_backtest():
     Trade.use_db = True
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_get_overall_performance(fee):
 
     create_mock_trades(fee)
@@ -1776,7 +1776,7 @@ def test_get_overall_performance(fee):
     assert 'count' in res[0]
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_get_best_pair(fee):
 
     res = Trade.get_best_pair()
@@ -1789,7 +1789,7 @@ def test_get_best_pair(fee):
     assert res[1] == 0.01
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_get_best_pair_lev(fee):
 
     res = Trade.get_best_pair()
@@ -1802,7 +1802,7 @@ def test_get_best_pair_lev(fee):
     assert res[1] == 0.1713156134055116
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_update_order_from_ccxt(caplog):
     # Most basic order return (only has orderid)
     o = Order.parse_from_ccxt_object({'id': '1234'}, 'ADA/USDT', 'buy')
@@ -1863,7 +1863,7 @@ def test_update_order_from_ccxt(caplog):
     Order.update_orders([o], {'id': '1234'})
 
 
-@ pytest.mark.usefixtures("init_persistence")
+@pytest.mark.usefixtures("init_persistence")
 def test_select_order(fee):
     create_mock_trades(fee)