Merge branch 'develop' into feat_readjust_entry

config_examples/config_binance.example.json

@@ -90,7 +90,7 @@
     },
     "bot_name": "freqtrade",
     "initial_state": "running",
-    "force_enter_enable": false,
+    "force_entry_enable": false,
     "internals": {
         "process_throttle_secs": 5
     }

docs/strategy-callbacks.md

@@ -7,6 +7,7 @@ Depending on the callback used, they may be called when entering / exiting a tra
 
 Currently available callbacks:
 
+* [`bot_start()`](#bot-start)
 * [`bot_loop_start()`](#bot-loop-start)
 * [`custom_stake_amount()`](#stake-size-management)
 * [`custom_exit()`](#custom-exit-signal)
@@ -22,6 +23,29 @@ Currently available callbacks:
 !!! Tip "Callback calling sequence"
     You can find the callback calling sequence in [bot-basics](bot-basics.md#bot-execution-logic)
 
+## Bot start
+
+A simple callback which is called once when the strategy is loaded.
+This can be used to perform actions that must only be performed once and runs after dataprovider and wallet are set
+
+``` python
+import requests
+
+class AwesomeStrategy(IStrategy):
+
+    # ... populate_* methods
+
+    def bot_start(self, **kwargs) -> None:
+        """
+        Called only once after bot instantiation.
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        """
+        if self.config['runmode'].value in ('live', 'dry_run'):
+            # Assign this to the class by using self.*
+            # can then be used by populate_* methods
+            self.cust_remote_data = requests.get('https://some_remote_source.example.com')
+
+```
 ## Bot loop start
 
 A simple callback which is called once at the start of every bot throttling iteration (roughly every 5 seconds, unless configured differently).

freqtrade/constants.py

@@ -459,8 +459,6 @@ SCHEMA_BACKTEST_REQUIRED = [
     'stake_currency',
     'stake_amount',
     'dry_run_wallet',
-    'stoploss',
-    'minimal_roi',
     'dataformat_ohlcv',
     'dataformat_trades',
 ]

freqtrade/data/btanalysis.py

@@ -12,7 +12,8 @@ import pandas as pd
 
 from freqtrade.constants import LAST_BT_RESULT_FN
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import get_backtest_metadata_filename, json_load
+from freqtrade.misc import json_load
+from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename
 from freqtrade.persistence import LocalTrade, Trade, init_db
 
 

freqtrade/freqtradebot.py

@@ -123,6 +123,8 @@ class FreqtradeBot(LoggingMixin):
                     self._schedule.every().day.at(t).do(update)
         self.last_process = datetime(1970, 1, 1, tzinfo=timezone.utc)
 
+        self.strategy.bot_start()
+
     def notify_status(self, msg: str) -> None:
         """
         Public method for users of this class (worker, etc.) to send notifications

freqtrade/misc.py

@@ -2,13 +2,11 @@
 Various tool function for Freqtrade and scripts
 """
 import gzip
-import hashlib
 import logging
 import re
-from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Iterator, List, Union
+from typing import Any, Iterator, List
 from typing.io import IO
 from urllib.parse import urlparse
 
@@ -251,34 +249,3 @@ def parse_db_uri_for_logging(uri: str):
         return uri
     pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0]
     return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
-
-
-def get_strategy_run_id(strategy) -> str:
-    """
-    Generate unique identification hash for a backtest run. Identical config and strategy file will
-    always return an identical hash.
-    :param strategy: strategy object.
-    :return: hex string id.
-    """
-    digest = hashlib.sha1()
-    config = deepcopy(strategy.config)
-
-    # Options that have no impact on results of individual backtest.
-    not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server')
-    for k in not_important_keys:
-        if k in config:
-            del config[k]
-
-    # Explicitly allow NaN values (e.g. max_open_trades).
-    # as it does not matter for getting the hash.
-    digest.update(rapidjson.dumps(config, default=str,
-                                  number_mode=rapidjson.NM_NAN).encode('utf-8'))
-    with open(strategy.__file__, 'rb') as fp:
-        digest.update(fp.read())
-    return digest.hexdigest().lower()
-
-
-def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path:
-    """Return metadata filename for specified backtest results file."""
-    filename = Path(filename)
-    return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}')

freqtrade/optimize/backtest_caching.py

@@ -0,0 +1,40 @@
+import hashlib
+from copy import deepcopy
+from pathlib import Path
+from typing import Union
+
+import rapidjson
+
+
+def get_strategy_run_id(strategy) -> str:
+    """
+    Generate unique identification hash for a backtest run. Identical config and strategy file will
+    always return an identical hash.
+    :param strategy: strategy object.
+    :return: hex string id.
+    """
+    digest = hashlib.sha1()
+    config = deepcopy(strategy.config)
+
+    # Options that have no impact on results of individual backtest.
+    not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server')
+    for k in not_important_keys:
+        if k in config:
+            del config[k]
+
+    # Explicitly allow NaN values (e.g. max_open_trades).
+    # as it does not matter for getting the hash.
+    digest.update(rapidjson.dumps(config, default=str,
+                                  number_mode=rapidjson.NM_NAN).encode('utf-8'))
+    # Include _ft_params_from_file - so changing parameter files cause cache eviction
+    digest.update(rapidjson.dumps(
+        strategy._ft_params_from_file, default=str, number_mode=rapidjson.NM_NAN).encode('utf-8'))
+    with open(strategy.__file__, 'rb') as fp:
+        digest.update(fp.read())
+    return digest.hexdigest().lower()
+
+
+def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path:
+    """Return metadata filename for specified backtest results file."""
+    filename = Path(filename)
+    return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}')

freqtrade/optimize/backtesting.py

@@ -24,8 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType
                              TradingMode)
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
-from freqtrade.misc import get_strategy_run_id
 from freqtrade.mixins import LoggingMixin
+from freqtrade.optimize.backtest_caching import get_strategy_run_id
 from freqtrade.optimize.bt_progress import BTProgress
 from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
                                                  store_backtest_signal_candles,
@@ -187,6 +187,7 @@ class Backtesting:
         # since a "perfect" stoploss-exit is assumed anyway
         # And the regular "stoploss" function would not apply to that case
         self.strategy.order_types['stoploss_on_exchange'] = False
+        self.strategy.bot_start()
 
     def _load_protections(self, strategy: IStrategy):
         if self.config.get('enable_protections', False):

freqtrade/optimize/edge_cli.py

@@ -44,6 +44,7 @@ class EdgeCli:
 
         self.edge._timerange = TimeRange.parse_timerange(None if self.config.get(
             'timerange') is None else str(self.config.get('timerange')))
+        self.strategy.bot_start()
 
     def start(self) -> None:
         result = self.edge.calculate(self.config['exchange']['pair_whitelist'])

freqtrade/optimize/optimize_reports.py

@@ -11,8 +11,8 @@ from tabulate import tabulate
 from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT
 from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change,
                                        calculate_max_drawdown)
-from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json,
-                            get_backtest_metadata_filename, round_coin_value)
+from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value
+from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename
 
 
 logger = logging.getLogger(__name__)

freqtrade/plot/plotting.py

@@ -610,6 +610,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
 
     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
     IStrategy.dp = DataProvider(config, exchange)
+    strategy.bot_start()
     plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count)
     timerange = plot_elements['timerange']
     trades = plot_elements['trades']

freqtrade/rpc/api_server/webserver.py

@@ -2,7 +2,7 @@ import logging
 from ipaddress import IPv4Address
 from typing import Any, Dict
 
-import rapidjson
+import orjson
 import uvicorn
 from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
@@ -24,7 +24,7 @@ class FTJSONResponse(JSONResponse):
         Use rapidjson for responses
         Handles NaN and Inf / -Inf in a javascript way by default.
         """
-        return rapidjson.dumps(content).encode("utf-8")
+        return orjson.dumps(content, option=orjson.OPT_SERIALIZE_NUMPY)
 
 
 class ApiServer(RPCHandler):

freqtrade/strategy/interface.py

@@ -193,6 +193,13 @@ class IStrategy(ABC, HyperStrategyMixin):
         """
         return self.populate_sell_trend(dataframe, metadata)
 
+    def bot_start(self, **kwargs) -> None:
+        """
+        Called only once after bot instantiation.
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        """
+        pass
+
     def bot_loop_start(self, **kwargs) -> None:
         """
         Called at the start of the bot iteration (one loop).

requirements.txt

@@ -27,6 +27,8 @@ py_find_1st==1.1.5
 
 # Load ticker files 30% faster
 python-rapidjson==1.6
+# Properly format api responses
+orjson==3.6.8
 
 # Notify systemd
 sdnotify==0.3.2

setup.py

@@ -57,6 +57,7 @@ setup(
         'pycoingecko',
         'py_find_1st',
         'python-rapidjson',
+        'orjson',
         'sdnotify',
         'colorama',
         'jinja2',

tests/optimize/test_backtesting.py

@@ -22,7 +22,7 @@ from freqtrade.data.history import get_timerange
 from freqtrade.enums import ExitType, RunMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange.exchange import timeframe_to_next_date
-from freqtrade.misc import get_strategy_run_id
+from freqtrade.optimize.backtest_caching import get_strategy_run_id
 from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.persistence import LocalTrade
 from freqtrade.resolvers import StrategyResolver
@@ -312,6 +312,7 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None:
     get_fee.assert_called()
     assert backtesting.fee == 0.5
     assert not backtesting.strategy.order_types["stoploss_on_exchange"]
+    assert backtesting.strategy.bot_started is True
 
 
 def test_backtesting_init_no_timeframe(mocker, default_conf, caplog) -> None:

tests/optimize/test_edge_cli.py

@@ -94,6 +94,7 @@ def test_edge_init(mocker, edge_conf) -> None:
     assert edge_cli.config == edge_conf
     assert edge_cli.config['stake_amount'] == 'unlimited'
     assert callable(edge_cli.edge.calculate)
+    assert edge_cli.strategy.bot_started is True
 
 
 def test_edge_init_fee(mocker, edge_conf) -> None:

tests/rpc/test_rpc_apiserver.py

@@ -13,7 +13,6 @@ import uvicorn
 from fastapi import FastAPI
 from fastapi.exceptions import HTTPException
 from fastapi.testclient import TestClient
-from numpy import isnan
 from requests.auth import _basic_auth_str
 
 from freqtrade.__init__ import __version__
@@ -985,7 +984,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short,
     assert_response(rc)
     resp_values = rc.json()
     assert len(resp_values) == 4
-    assert isnan(resp_values[0]['profit_abs'])
+    assert resp_values[0]['profit_abs'] is None
 
 
 def test_api_version(botclient):

tests/strategy/strats/strategy_test_v3.py

@@ -82,6 +82,11 @@ class StrategyTestV3(IStrategy):
     #         })
     #     return prot
 
+    bot_started = False
+
+    def bot_start(self):
+        self.bot_started = True
+
     def informative_pairs(self):
 
         return []