From 97a6fb285f894ba91b40dc8fa51d28d750d894b0 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 10 Jan 2023 17:52:24 -0700 Subject: [PATCH 01/56] revert to dataframe.to_json --- freqtrade/misc.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 93e8da6dd..7b9ff1f1d 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -10,7 +10,6 @@ from typing import Any, Dict, Iterator, List, Mapping, Union from typing.io import IO from urllib.parse import urlparse -import orjson import pandas as pd import rapidjson @@ -263,15 +262,7 @@ def dataframe_to_json(dataframe: pd.DataFrame) -> str: :param dataframe: A pandas DataFrame :returns: A JSON string of the pandas DataFrame """ - # https://github.com/pandas-dev/pandas/issues/24889 - # https://github.com/pandas-dev/pandas/issues/40443 - # We need to convert to a dict to avoid mem leak - def default(z): - if isinstance(z, pd.Timestamp): - return z.timestamp() * 1e3 - raise TypeError - - return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8') + return dataframe.to_json(orient='split') def json_to_dataframe(data: str) -> pd.DataFrame: From 3a7e41e1777dc4835b7d1e3ed35318d829f88844 Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Wed, 10 May 2023 10:32:00 +0100 Subject: [PATCH 02/56] Update rest_client.py Add fix for forceenter to avoid passing None prices back to the API --- scripts/rest_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ccffe7f5f..d301d06e6 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -279,8 +279,9 @@ class FtRestClient(): """ data = {"pair": pair, "side": side, - "price": price, } + if price: + params['price'] = price return self._post("forceenter", data=data) def forceexit(self, tradeid, ordertype=None, amount=None): From 242247be47a629efef0d412e8b27a28e5aae944f Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Wed, 10 May 2023 10:56:14 +0100 Subject: [PATCH 03/56] Fix var name --- scripts/rest_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rest_client.py b/scripts/rest_client.py index d301d06e6..0772af269 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -281,7 +281,7 @@ class FtRestClient(): "side": side, } if price: - params['price'] = price + data['price'] = price return self._post("forceenter", data=data) def forceexit(self, tradeid, ordertype=None, amount=None): From d3382fbe045872283d3a88b2a609dc63f02b0288 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 09:04:37 +0200 Subject: [PATCH 04/56] Reduce usage of arrow --- freqtrade/wallets.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 9a33d1fb1..895e8dd8c 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -3,10 +3,9 @@ import logging from copy import deepcopy +from datetime import datetime from typing import Dict, NamedTuple, Optional -import arrow - from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, Config from freqtrade.enums import RunMode, TradingMode from freqtrade.exceptions import DependencyException @@ -166,14 +165,15 @@ class Wallets: for trading operations, the latest balance is needed. :param require_update: Allow skipping an update if balances were recently refreshed """ - if (require_update or (self._last_wallet_refresh + 3600 < arrow.utcnow().int_timestamp)): + now = datetime.now().timestamp() + if (require_update or (self._last_wallet_refresh + 3600 < now)): if (not self._config['dry_run'] or self._config.get('runmode') == RunMode.LIVE): self._update_live() else: self._update_dry() if self._log: logger.info('Wallets synced.') - self._last_wallet_refresh = arrow.utcnow().int_timestamp + self._last_wallet_refresh = datetime.now().timestamp() def get_all_balances(self) -> Dict[str, Wallet]: return self._wallets From 1d03e8bc5f0fa6dff4c01e7e87809d12ff76af52 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 09:08:31 +0200 Subject: [PATCH 05/56] Reduce arrow usage further --- freqtrade/data/history/history_utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index b567b58bf..dc3c7c1e6 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -1,10 +1,9 @@ import logging import operator -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from typing import Dict, List, Optional, Tuple -import arrow from pandas import DataFrame, concat from freqtrade.configuration import TimeRange @@ -236,8 +235,8 @@ def _download_pair_history(pair: str, *, new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms if since_ms else - arrow.utcnow().shift( - days=-new_pairs_days).int_timestamp * 1000, + int((datetime.now() - timedelta(days=new_pairs_days) + ).timestamp()) * 1000, is_new_pair=data.empty, candle_type=candle_type, until_ms=until_ms if until_ms else None @@ -349,7 +348,7 @@ def _download_trades_history(exchange: Exchange, trades = [] if not since: - since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 + since = int((datetime.now() - timedelta(days=-new_pairs_days)).timestamp()) * 1000 from_id = trades[-1][1] if trades else None if trades and since < trades[-1][0]: From 2477ef57f992083a6c60d896e2488d5366209556 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 09:13:25 +0200 Subject: [PATCH 06/56] Reduce arrow usage throughout code --- freqtrade/edge/edge_positioning.py | 5 +++-- freqtrade/exchange/binance.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 73820ecbe..6f493b72b 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -3,6 +3,7 @@ import logging from collections import defaultdict from copy import deepcopy +from datetime import datetime, timezone from typing import Any, Dict, List, NamedTuple import arrow @@ -97,7 +98,7 @@ class Edge: heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated > 0) and ( - self._last_updated + heartbeat > arrow.utcnow().int_timestamp): + self._last_updated + heartbeat > int(datetime.now(timezone.utc).timestamp())): return False data: Dict[str, Any] = {} @@ -189,7 +190,7 @@ class Edge: # Fill missing, calculable columns, profit, duration , abs etc. trades_df = self._fill_calculable_fields(DataFrame(trades)) self._cached_pairs = self._process_expectancy(trades_df) - self._last_updated = arrow.utcnow().int_timestamp + self._last_updated = int(datetime.now(timezone.utc).timestamp()) return True diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 7ac496f62..caca3eefb 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -1,10 +1,9 @@ """ Binance exchange subclass """ import logging -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Optional, Tuple -import arrow import ccxt from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode @@ -105,8 +104,9 @@ class Binance(Exchange): if x and x[3] and x[3][0] and x[3][0][0] > since_ms: # Set starting date to first available candle. since_ms = x[3][0][0] - logger.info(f"Candle-data for {pair} available starting with " - f"{arrow.get(since_ms // 1000).isoformat()}.") + logger.info( + f"Candle-data for {pair} available starting with " + f"{datetime.fromtimestamp(since_ms // 1000, tz=timezone.utc).isoformat()}.") return await super()._async_get_historic_ohlcv( pair=pair, From 6044bbb6b1b9a343edb44cc504d695707b19701f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 09:28:59 +0200 Subject: [PATCH 07/56] Add datetime helpers to unify code --- freqtrade/util/__init__.py | 13 +++++++++++-- freqtrade/util/datetime_helpers.py | 17 ++++++++++++++++ tests/utils/test_datetime_helpers.py | 29 ++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 freqtrade/util/datetime_helpers.py create mode 100644 tests/utils/test_datetime_helpers.py diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 3c3c034c1..37d95e511 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,2 +1,11 @@ -from freqtrade.util.ft_precise import FtPrecise # noqa: F401 -from freqtrade.util.periodic_cache import PeriodicCache # noqa: F401 +from freqtrade.util.datetime_helpers import dt_from_ts, dt_now +from freqtrade.util.ft_precise import FtPrecise +from freqtrade.util.periodic_cache import PeriodicCache + + +__all__ = [ + 'dt_from_ts', + 'dt_now', + 'FtPrecise', + 'PeriodicCache', +] diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py new file mode 100644 index 000000000..aa9403cba --- /dev/null +++ b/freqtrade/util/datetime_helpers.py @@ -0,0 +1,17 @@ +from datetime import datetime, timezone + + +def dt_now() -> datetime: + """Return the current datetime in UTC.""" + return datetime.now(timezone.utc) + + +def dt_from_ts(timestamp: float) -> datetime: + """ + Return a datetime from a timestamp. + :param timestamp: timestamp in seconds or milliseconds + """ + if timestamp > 1e10: + # Timezone in ms - convert to seconds + timestamp /= 1000 + return datetime.fromtimestamp(timestamp, tz=timezone.utc) diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py new file mode 100644 index 000000000..947aab1b8 --- /dev/null +++ b/tests/utils/test_datetime_helpers.py @@ -0,0 +1,29 @@ +from datetime import datetime, timedelta, timezone + +import pytest +import time_machine + +from freqtrade.util import dt_from_ts, dt_now + + +def test_dt_now(): + with time_machine.travel("2021-09-01 05:01:00 +00:00", tick=False) as t: + now = datetime.now(timezone.utc) + assert dt_now() == now + + t.shift(timedelta(hours=5)) + assert dt_now() >= now + assert dt_now() == datetime.now(timezone.utc) + + +@pytest.mark.parametrize('as_ms', [True, False]) +def test_dt_from_ts(as_ms): + multi = 1000 if as_ms else 1 + assert dt_from_ts(1683244800.0 * multi) == datetime(2023, 5, 5, tzinfo=timezone.utc) + assert dt_from_ts(1683244800.5555 * multi) == datetime(2023, 5, 5, 0, 0, 0, 555500, + tzinfo=timezone.utc) + # As int + assert dt_from_ts(1683244800 * multi) == datetime(2023, 5, 5, tzinfo=timezone.utc) + # As milliseconds + assert dt_from_ts(1683244800 * multi) == datetime(2023, 5, 5, tzinfo=timezone.utc) + assert dt_from_ts(1683242400 * multi) == datetime(2023, 5, 4, 23, 20, tzinfo=timezone.utc) From 6b735bc683167b9f3c7d62becdf0429e5f81641d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 10:32:18 +0200 Subject: [PATCH 08/56] Implement dt_now --- freqtrade/strategy/interface.py | 14 +++++++------- tests/strategy/test_interface.py | 11 ++++++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7adb7a154..80f74bfb6 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -7,7 +7,6 @@ from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional, Tuple, Union -import arrow from pandas import DataFrame from freqtrade.constants import CUSTOM_TAG_MAX_LENGTH, Config, IntOrInf, ListPairsWithTimeframes @@ -23,6 +22,7 @@ from freqtrade.strategy.informative_decorator import (InformativeData, PopulateI _create_and_merge_informative_pair, _format_pair_name) from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper +from freqtrade.util import dt_now from freqtrade.wallets import Wallets @@ -938,7 +938,7 @@ class IStrategy(ABC, HyperStrategyMixin): pair: str, timeframe: str, dataframe: DataFrame, - ) -> Tuple[Optional[DataFrame], Optional[arrow.Arrow]]: + ) -> Tuple[Optional[DataFrame], Optional[datetime]]: """ Calculates current signal based based on the entry order or exit order columns of the dataframe. @@ -955,15 +955,15 @@ class IStrategy(ABC, HyperStrategyMixin): latest_date = dataframe['date'].max() latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1] # Explicitly convert to arrow object to ensure the below comparison does not fail - latest_date = arrow.get(latest_date) + latest_date = latest_date.to_pydatetime() # Check if dataframe is out of date timeframe_minutes = timeframe_to_minutes(timeframe) offset = self.config.get('exchange', {}).get('outdated_offset', 5) - if latest_date < (arrow.utcnow().shift(minutes=-(timeframe_minutes * 2 + offset))): + if latest_date < (dt_now() - timedelta(minutes=timeframe_minutes * 2 + offset)): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', - pair, int((arrow.utcnow() - latest_date).total_seconds() // 60) + pair, int((dt_now() - latest_date).total_seconds() // 60) ) return None, None return latest, latest_date @@ -1046,8 +1046,8 @@ class IStrategy(ABC, HyperStrategyMixin): timeframe_seconds = timeframe_to_seconds(timeframe) if self.ignore_expired_candle( - latest_date=latest_date.datetime, - current_time=datetime.now(timezone.utc), + latest_date=latest_date, + current_time=dt_now(), timeframe_seconds=timeframe_seconds, enter=bool(enter_signal) ): diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 204fa996d..ef8fd0be9 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -22,6 +22,7 @@ from freqtrade.strategy.hyper import detect_parameters from freqtrade.strategy.parameters import (BaseParameter, BooleanParameter, CategoricalParameter, DecimalParameter, IntParameter, RealParameter) from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper +from freqtrade.util import dt_now from tests.conftest import (CURRENT_TEST_STRATEGY, TRADE_SIDES, create_mock_trades, log_has, log_has_re) @@ -34,7 +35,7 @@ _STRATEGY.dp = DataProvider({}, None, None) def test_returns_latest_signal(ohlcv_history): - ohlcv_history.loc[1, 'date'] = arrow.utcnow() + ohlcv_history.loc[1, 'date'] = dt_now() # Take a copy to correctly modify the call mocked_history = ohlcv_history.copy() mocked_history['enter_long'] = 0 @@ -159,7 +160,7 @@ def test_get_signal_exception_valueerror(mocker, caplog, ohlcv_history): def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history): # default_conf defines a 5m interval. we check interval * 2 + 5m # this is necessary as the last candle is removed (partial candles) by default - ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16) + ohlcv_history.loc[1, 'date'] = dt_now() - timedelta(minutes=16) # Take a copy to correctly modify the call mocked_history = ohlcv_history.copy() mocked_history['exit_long'] = 0 @@ -180,7 +181,7 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history): def test_get_signal_no_sell_column(default_conf, mocker, caplog, ohlcv_history): # default_conf defines a 5m interval. we check interval * 2 + 5m # this is necessary as the last candle is removed (partial candles) by default - ohlcv_history.loc[1, 'date'] = arrow.utcnow() + ohlcv_history.loc[1, 'date'] = dt_now() # Take a copy to correctly modify the call mocked_history = ohlcv_history.copy() # Intentionally don't set sell column @@ -224,7 +225,7 @@ def test_ignore_expired_candle(default_conf): def test_assert_df_raise(mocker, caplog, ohlcv_history): - ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16) + ohlcv_history.loc[1, 'date'] = dt_now() - timedelta(minutes=16) # Take a copy to correctly modify the call mocked_history = ohlcv_history.copy() mocked_history['sell'] = 0 @@ -323,7 +324,7 @@ def test_min_roi_reached(default_conf, fee) -> None: pair='ETH/BTC', stake_amount=0.001, amount=5, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', From 261df527d97150ece115fe8f199edbc96fdf121c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 10:34:06 +0200 Subject: [PATCH 09/56] dt_now --- freqtrade/edge/edge_positioning.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 6f493b72b..af66345e0 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -3,10 +3,9 @@ import logging from collections import defaultdict from copy import deepcopy -from datetime import datetime, timezone +from datetime import timedelta from typing import Any, Dict, List, NamedTuple -import arrow import numpy as np import utils_find_1st as utf1st from pandas import DataFrame @@ -19,6 +18,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.strategy.interface import IStrategy +from freqtrade.util import dt_now logger = logging.getLogger(__name__) @@ -80,8 +80,8 @@ class Edge: self._stoploss_range_step ) - self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift( - days=-1 * self._since_number_of_days).format('YYYYMMDD')) + self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % ( + dt_now() - timedelta(days=self._since_number_of_days)).strftime('YYYYMMDD')) if config.get('fee'): self.fee = config['fee'] else: @@ -98,7 +98,7 @@ class Edge: heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated > 0) and ( - self._last_updated + heartbeat > int(datetime.now(timezone.utc).timestamp())): + self._last_updated + heartbeat > int(dt_now().timestamp())): return False data: Dict[str, Any] = {} @@ -190,7 +190,7 @@ class Edge: # Fill missing, calculable columns, profit, duration , abs etc. trades_df = self._fill_calculable_fields(DataFrame(trades)) self._cached_pairs = self._process_expectancy(trades_df) - self._last_updated = int(datetime.now(timezone.utc).timestamp()) + self._last_updated = int(dt_now().timestamp()) return True From 5c6f3ea439fec82342b71246fce253cd71619536 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 10:37:16 +0200 Subject: [PATCH 10/56] Improve wallets time handling --- freqtrade/wallets.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 895e8dd8c..da64515a4 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -3,7 +3,7 @@ import logging from copy import deepcopy -from datetime import datetime +from datetime import datetime, timedelta from typing import Dict, NamedTuple, Optional from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, Config @@ -12,6 +12,7 @@ from freqtrade.exceptions import DependencyException from freqtrade.exchange import Exchange from freqtrade.misc import safe_value_fallback from freqtrade.persistence import LocalTrade, Trade +from freqtrade.util.datetime_helpers import dt_now logger = logging.getLogger(__name__) @@ -42,7 +43,7 @@ class Wallets: self._wallets: Dict[str, Wallet] = {} self._positions: Dict[str, PositionWallet] = {} self.start_cap = config['dry_run_wallet'] - self._last_wallet_refresh = 0 + self._last_wallet_refresh: Optional[datetime] = None self.update() def get_free(self, currency: str) -> float: @@ -165,15 +166,19 @@ class Wallets: for trading operations, the latest balance is needed. :param require_update: Allow skipping an update if balances were recently refreshed """ - now = datetime.now().timestamp() - if (require_update or (self._last_wallet_refresh + 3600 < now)): + now = dt_now() + if ( + require_update + or self._last_wallet_refresh is None + or (self._last_wallet_refresh + timedelta(seconds=3600) < now) + ): if (not self._config['dry_run'] or self._config.get('runmode') == RunMode.LIVE): self._update_live() else: self._update_dry() if self._log: logger.info('Wallets synced.') - self._last_wallet_refresh = datetime.now().timestamp() + self._last_wallet_refresh = dt_now() def get_all_balances(self) -> Dict[str, Wallet]: return self._wallets From aa949153eb30b0e99c53f81a8a76ae6f18e04cd6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 10:43:06 +0200 Subject: [PATCH 11/56] Add now ts helper --- freqtrade/util/__init__.py | 3 ++- freqtrade/util/datetime_helpers.py | 5 +++++ tests/utils/test_datetime_helpers.py | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 37d95e511..f1759f27e 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,4 +1,4 @@ -from freqtrade.util.datetime_helpers import dt_from_ts, dt_now +from freqtrade.util.datetime_helpers import dt_from_ts, dt_now, dt_now_ts from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache @@ -6,6 +6,7 @@ from freqtrade.util.periodic_cache import PeriodicCache __all__ = [ 'dt_from_ts', 'dt_now', + 'dt_now_ts', 'FtPrecise', 'PeriodicCache', ] diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index aa9403cba..6f169e7e5 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -6,6 +6,11 @@ def dt_now() -> datetime: return datetime.now(timezone.utc) +def dt_now_ts() -> int: + """Return the current timestamp in ms as a timestamp in UTC.""" + return int(dt_now().timestamp() * 1000) + + def dt_from_ts(timestamp: float) -> datetime: """ Return a datetime from a timestamp. diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index 947aab1b8..7b7e2b7c6 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -3,17 +3,19 @@ from datetime import datetime, timedelta, timezone import pytest import time_machine -from freqtrade.util import dt_from_ts, dt_now +from freqtrade.util import dt_from_ts, dt_now, dt_now_ts def test_dt_now(): with time_machine.travel("2021-09-01 05:01:00 +00:00", tick=False) as t: now = datetime.now(timezone.utc) assert dt_now() == now + assert dt_now_ts() == int(now.timestamp() * 1000) t.shift(timedelta(hours=5)) assert dt_now() >= now assert dt_now() == datetime.now(timezone.utc) + assert dt_now_ts() == int(dt_now().timestamp() * 1000) @pytest.mark.parametrize('as_ms', [True, False]) From 000f72942a4ab6bd7e0d960f18e0e7a5e75fc661 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 10:45:07 +0200 Subject: [PATCH 12/56] Improve dt_now_ts helper --- freqtrade/util/__init__.py | 4 ++-- freqtrade/util/datetime_helpers.py | 5 ++++- tests/utils/test_datetime_helpers.py | 11 ++++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index f1759f27e..67cdafc3c 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,4 +1,4 @@ -from freqtrade.util.datetime_helpers import dt_from_ts, dt_now, dt_now_ts +from freqtrade.util.datetime_helpers import dt_from_ts, dt_now, dt_ts from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache @@ -6,7 +6,7 @@ from freqtrade.util.periodic_cache import PeriodicCache __all__ = [ 'dt_from_ts', 'dt_now', - 'dt_now_ts', + 'dt_ts', 'FtPrecise', 'PeriodicCache', ] diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index 6f169e7e5..9e7f0a8a8 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -1,4 +1,5 @@ from datetime import datetime, timezone +from typing import Optional def dt_now() -> datetime: @@ -6,8 +7,10 @@ def dt_now() -> datetime: return datetime.now(timezone.utc) -def dt_now_ts() -> int: +def dt_ts(dt: Optional[datetime] = None) -> int: """Return the current timestamp in ms as a timestamp in UTC.""" + if dt: + return int(dt.timestamp() * 1000) return int(dt_now().timestamp() * 1000) diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index 7b7e2b7c6..47487a04b 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -3,19 +3,24 @@ from datetime import datetime, timedelta, timezone import pytest import time_machine -from freqtrade.util import dt_from_ts, dt_now, dt_now_ts +from freqtrade.util import dt_from_ts, dt_now, dt_ts def test_dt_now(): with time_machine.travel("2021-09-01 05:01:00 +00:00", tick=False) as t: now = datetime.now(timezone.utc) assert dt_now() == now - assert dt_now_ts() == int(now.timestamp() * 1000) + assert dt_ts() == int(now.timestamp() * 1000) + assert dt_ts(now) == int(now.timestamp() * 1000) t.shift(timedelta(hours=5)) assert dt_now() >= now assert dt_now() == datetime.now(timezone.utc) - assert dt_now_ts() == int(dt_now().timestamp() * 1000) + assert dt_ts() == int(dt_now().timestamp() * 1000) + # Test with different time than now + assert dt_ts(now) == int(now.timestamp() * 1000) + + @pytest.mark.parametrize('as_ms', [True, False]) From 55ce58d79f8bda2547eeca03bc23751e875644b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 10:48:04 +0200 Subject: [PATCH 13/56] Reduce some arrow usages in favor of dt helpers --- freqtrade/exchange/exchange.py | 22 ++++++++++++---------- tests/exchange/test_exchange.py | 5 +++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index bb9c7c1b3..1c287438d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -42,6 +42,8 @@ from freqtrade.exchange.types import OHLCVResponse, OrderBook, Ticker, Tickers from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json, safe_value_fallback2) from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist +from freqtrade.util import dt_from_ts, dt_now +from freqtrade.util.datetime_helpers import dt_ts logger = logging.getLogger(__name__) @@ -490,7 +492,7 @@ class Exchange: try: self._markets = self._api.load_markets(params={}) self._load_async_markets() - self._last_markets_refresh = arrow.utcnow().int_timestamp + self._last_markets_refresh = dt_ts() if self._ft_has['needs_trading_fees']: self._trading_fees = self.fetch_trading_fees() @@ -501,15 +503,14 @@ class Exchange: """Reload markets both sync and async if refresh interval has passed """ # Check whether markets have to be reloaded if (self._last_markets_refresh > 0) and ( - self._last_markets_refresh + self.markets_refresh_interval - > arrow.utcnow().int_timestamp): + self._last_markets_refresh + self.markets_refresh_interval > dt_ts()): return None logger.debug("Performing scheduled market reload..") try: self._markets = self._api.load_markets(reload=True, params={}) # Also reload async markets to avoid issues with newly listed pairs self._load_async_markets(reload=True) - self._last_markets_refresh = arrow.utcnow().int_timestamp + self._last_markets_refresh = dt_ts() self.fill_leverage_tiers() except ccxt.BaseError: logger.exception("Could not reload markets.") @@ -843,7 +844,8 @@ class Exchange: def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, leverage: float, params: Dict = {}, stop_loss: bool = False) -> Dict[str, Any]: - order_id = f'dry_run_{side}_{datetime.now().timestamp()}' + now = dt_now() + order_id = f'dry_run_{side}_{now.timestamp()}' # Rounding here must respect to contract sizes _amount = self._contracts_to_amount( pair, self.amount_to_precision(pair, self._amount_to_contracts(pair, amount))) @@ -858,8 +860,8 @@ class Exchange: 'side': side, 'filled': 0, 'remaining': _amount, - 'datetime': arrow.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ'), - 'timestamp': arrow.utcnow().int_timestamp * 1000, + 'datetime': now.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), + 'timestamp': dt_ts(now), 'status': "open", 'fee': None, 'info': {}, @@ -1934,7 +1936,7 @@ class Exchange: ) input_coroutines = [self._async_get_candle_history( pair, timeframe, candle_type, since) for since in - range(since_ms, until_ms or (arrow.utcnow().int_timestamp * 1000), one_call)] + range(since_ms, until_ms or dt_ts(), one_call)] data: List = [] # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling @@ -2117,7 +2119,7 @@ class Exchange: """ try: # Fetch OHLCV asynchronously - s = '(' + arrow.get(since_ms // 1000).isoformat() + ') ' if since_ms is not None else '' + s = '(' + dt_from_ts(since_ms).isoformat() + ') ' if since_ms is not None else '' logger.debug( "Fetching pair %s, %s, interval %s, since %s %s...", pair, candle_type, timeframe, since_ms, s @@ -2207,7 +2209,7 @@ class Exchange: logger.debug( "Fetching trades for pair %s, since %s %s...", pair, since, - '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else '' + '(' + dt_from_ts(since).isoformat() + ') ' if since is not None else '' ) trades = await self._api_async.fetch_trades(pair, since=since, limit=1000) trades = self._trades_contracts_to_amount(trades) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ebbecdad0..7d4fbfb1a 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -23,6 +23,7 @@ from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_CO calculate_backoff, remove_credentials) from freqtrade.exchange.exchange import amount_to_contract_precision from freqtrade.resolvers.exchange_resolver import ExchangeResolver +from freqtrade.util import dt_now, dt_ts from tests.conftest import (EXMS, generate_test_data_raw, get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re) @@ -646,7 +647,7 @@ def test_reload_markets(default_conf, mocker, caplog): exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance", mock_markets=False) exchange._load_async_markets = MagicMock() - exchange._last_markets_refresh = arrow.utcnow().int_timestamp + exchange._last_markets_refresh = dt_ts() assert exchange.markets == initial_markets @@ -657,7 +658,7 @@ def test_reload_markets(default_conf, mocker, caplog): api_mock.load_markets = MagicMock(return_value=updated_markets) # more than 10 minutes have passed, reload is executed - exchange._last_markets_refresh = arrow.utcnow().int_timestamp - 15 * 60 + exchange._last_markets_refresh = dt_ts(dt_now() - timedelta(minutes=15)) exchange.reload_markets() assert exchange.markets == updated_markets assert exchange._load_async_markets.call_count == 1 From 7f73e99437a36d27fe33716948f481acb04d47a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 11:01:50 +0200 Subject: [PATCH 14/56] Simplify exchange_utils --- freqtrade/exchange/exchange_utils.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/exchange_utils.py b/freqtrade/exchange/exchange_utils.py index 83d2a214d..c6c2d5a24 100644 --- a/freqtrade/exchange/exchange_utils.py +++ b/freqtrade/exchange/exchange_utils.py @@ -11,6 +11,7 @@ from ccxt import (DECIMAL_PLACES, ROUND, ROUND_DOWN, ROUND_UP, SIGNIFICANT_DIGIT from freqtrade.exchange.common import BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED from freqtrade.util import FtPrecise +from freqtrade.util.datetime_helpers import dt_from_ts, dt_ts CcxtModuleType = Any @@ -99,9 +100,8 @@ def timeframe_to_prev_date(timeframe: str, date: Optional[datetime] = None) -> d if not date: date = datetime.now(timezone.utc) - new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000, - ROUND_DOWN) // 1000 - return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) + new_timestamp = ccxt.Exchange.round_timeframe(timeframe, dt_ts(date), ROUND_DOWN) // 1000 + return dt_from_ts(new_timestamp) def timeframe_to_next_date(timeframe: str, date: Optional[datetime] = None) -> datetime: @@ -113,9 +113,8 @@ def timeframe_to_next_date(timeframe: str, date: Optional[datetime] = None) -> d """ if not date: date = datetime.now(timezone.utc) - new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000, - ROUND_UP) // 1000 - return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) + new_timestamp = ccxt.Exchange.round_timeframe(timeframe, dt_ts(date), ROUND_UP) // 1000 + return dt_from_ts(new_timestamp) def date_minus_candles( From 5b66ef4bea6ad222033e5d4e67bea59c7ecea7c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 11:02:11 +0200 Subject: [PATCH 15/56] Implement datetime.floor --- freqtrade/util/__init__.py | 3 ++- freqtrade/util/datetime_helpers.py | 10 +++++++++- tests/utils/test_datetime_helpers.py | 10 +++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 67cdafc3c..b28544fd3 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,4 +1,4 @@ -from freqtrade.util.datetime_helpers import dt_from_ts, dt_now, dt_ts +from freqtrade.util.datetime_helpers import dt_floor_day, dt_from_ts, dt_now, dt_ts from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache @@ -7,6 +7,7 @@ __all__ = [ 'dt_from_ts', 'dt_now', 'dt_ts', + 'dt_floor_day', 'FtPrecise', 'PeriodicCache', ] diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index 9e7f0a8a8..228bca185 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -8,12 +8,20 @@ def dt_now() -> datetime: def dt_ts(dt: Optional[datetime] = None) -> int: - """Return the current timestamp in ms as a timestamp in UTC.""" + """ + Return dt in ms as a timestamp in UTC. + If dt is None, return the current datetime in UTC. + """ if dt: return int(dt.timestamp() * 1000) return int(dt_now().timestamp() * 1000) +def dt_floor_day(dt: datetime) -> datetime: + """Return the floor of the day for the given datetime.""" + return dt.replace(hour=0, minute=0, second=0, microsecond=0) + + def dt_from_ts(timestamp: float) -> datetime: """ Return a datetime from a timestamp. diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index 47487a04b..7cff44847 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta, timezone import pytest import time_machine -from freqtrade.util import dt_from_ts, dt_now, dt_ts +from freqtrade.util import dt_floor_day, dt_from_ts, dt_now, dt_ts def test_dt_now(): @@ -21,8 +21,6 @@ def test_dt_now(): assert dt_ts(now) == int(now.timestamp() * 1000) - - @pytest.mark.parametrize('as_ms', [True, False]) def test_dt_from_ts(as_ms): multi = 1000 if as_ms else 1 @@ -34,3 +32,9 @@ def test_dt_from_ts(as_ms): # As milliseconds assert dt_from_ts(1683244800 * multi) == datetime(2023, 5, 5, tzinfo=timezone.utc) assert dt_from_ts(1683242400 * multi) == datetime(2023, 5, 4, 23, 20, tzinfo=timezone.utc) + + +def test_dt_floor_day(): + now = datetime(2023, 9, 1, 5, 2, 3, 455555, tzinfo=timezone.utc) + + assert dt_floor_day(now) == datetime(2023, 9, 1, tzinfo=timezone.utc) From e4f701fd0d763864960f2905bfe44b16691e6f0b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 11:06:15 +0200 Subject: [PATCH 16/56] Don't use arrow for everything --- freqtrade/plugins/pairlist/AgeFilter.py | 13 +++++-------- freqtrade/plugins/pairlist/VolatilityFilter.py | 8 +++----- freqtrade/plugins/pairlist/VolumePairList.py | 7 ++++--- freqtrade/plugins/pairlist/rangestabilityfilter.py | 8 +++----- 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index f9c02e250..2af86592f 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -3,9 +3,9 @@ Minimum age (days listed) pair list filter """ import logging from copy import deepcopy +from datetime import timedelta from typing import Any, Dict, List, Optional -import arrow from pandas import DataFrame from freqtrade.constants import Config, ListPairsWithTimeframes @@ -13,7 +13,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList -from freqtrade.util import PeriodicCache +from freqtrade.util import PeriodicCache, dt_floor_day, dt_now, dt_ts logger = logging.getLogger(__name__) @@ -84,10 +84,7 @@ class AgeFilter(IPairList): since_days = -( self._max_days_listed if self._max_days_listed else self._min_days_listed ) - 1 - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=since_days) - .float_timestamp) * 1000 + since_ms = dt_ts(dt_floor_day(dt_now()) + timedelta(days=since_days)) candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) if self._enabled: for p in deepcopy(pairlist): @@ -116,7 +113,7 @@ class AgeFilter(IPairList): ): # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol - self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000 + self._symbolsChecked[pair] = dt_ts() return True else: self.log_once(( @@ -127,6 +124,6 @@ class AgeFilter(IPairList): " or more than " f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else ''), logger.info) - self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000 + self._symbolsCheckFailed[pair] = dt_ts() return False return False diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 401a2e86c..9196026bb 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -4,9 +4,9 @@ Volatility pairlist filter import logging import sys from copy import deepcopy +from datetime import timedelta from typing import Any, Dict, List, Optional -import arrow import numpy as np from cachetools import TTLCache from pandas import DataFrame @@ -16,6 +16,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.util import dt_floor_day, dt_now, dt_ts logger = logging.getLogger(__name__) @@ -73,10 +74,7 @@ class VolatilityFilter(IPairList): needed_pairs: ListPairsWithTimeframes = [ (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache] - since_ms = (arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .int_timestamp) * 1000 + since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days - 1)) # Get all candles candles = {} if needed_pairs: diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 2649a8425..b9c312f87 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -4,7 +4,7 @@ Volume PairList provider Provides dynamic pair list based on trade volumes """ import logging -from datetime import datetime, timedelta, timezone +from datetime import timedelta from typing import Any, Dict, List, Literal from cachetools import TTLCache @@ -15,6 +15,7 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers from freqtrade.misc import format_ms_time from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.util import dt_now logger = logging.getLogger(__name__) @@ -161,13 +162,13 @@ class VolumePairList(IPairList): # get lookback period in ms, for exchange ohlcv fetch since_ms = int(timeframe_to_prev_date( self._lookback_timeframe, - datetime.now(timezone.utc) + timedelta( + dt_now() + timedelta( minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min) ).timestamp()) * 1000 to_ms = int(timeframe_to_prev_date( self._lookback_timeframe, - datetime.now(timezone.utc) - timedelta(minutes=self._tf_in_min) + dt_now() - timedelta(minutes=self._tf_in_min) ).timestamp()) * 1000 # todo: utc date output for starting date diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 546b026cb..1181b2812 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -3,9 +3,9 @@ Rate of change pairlist filter """ import logging from copy import deepcopy +from datetime import timedelta from typing import Any, Dict, List, Optional -import arrow from cachetools import TTLCache from pandas import DataFrame @@ -14,6 +14,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.util import dt_floor_day, dt_now, dt_ts logger = logging.getLogger(__name__) @@ -71,10 +72,7 @@ class RangeStabilityFilter(IPairList): needed_pairs: ListPairsWithTimeframes = [ (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache] - since_ms = (arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .int_timestamp) * 1000 + since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days - 1)) # Get all candles candles = {} if needed_pairs: From cfae98ae0066d18239678503392bab1ba2d98e5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 11:10:21 +0200 Subject: [PATCH 17/56] dt_now for tests --- tests/persistence/test_persistence.py | 13 +++++++------ tests/strategy/test_interface.py | 6 +++--- tests/test_freqtradebot.py | 19 ++++++++++--------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 6af629c75..92a0592cb 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -10,6 +10,7 @@ from freqtrade.constants import CUSTOM_TAG_MAX_LENGTH, DATETIME_PRINT_FORMAT from freqtrade.enums import TradingMode from freqtrade.exceptions import DependencyException from freqtrade.persistence import LocalTrade, Order, Trade, init_db +from freqtrade.util.datetime_helpers import dt_now from tests.conftest import create_mock_trades, create_mock_trades_with_leverage, log_has, log_has_re @@ -27,7 +28,7 @@ def test_enter_exit_side(fee, is_short): open_rate=0.01, amount=5, is_open=True, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -49,7 +50,7 @@ def test_set_stop_loss_liquidation(fee): open_rate=2.0, amount=30.0, is_open=True, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -329,7 +330,7 @@ def test_borrowed(fee, is_short, lev, borrowed, trading_mode): open_rate=2.0, amount=30.0, is_open=True, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -428,7 +429,7 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ open_rate=open_rate, amount=30.0, is_open=True, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -485,7 +486,7 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, is_open=True, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), exchange='binance', trading_mode=margin, leverage=1.0, @@ -2592,7 +2593,7 @@ def test_recalc_trade_from_orders_dca(data) -> None: open_rate=data['orders'][0][0][2], amount=data['orders'][0][0][1], is_open=True, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=data['fee'], fee_close=data['fee'], exchange='binance', diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index ef8fd0be9..a7dde20d6 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -465,7 +465,7 @@ def test_ft_stoploss_reached(default_conf, fee, profit, adjusted, expected, liq, if custom_stop: strategy.custom_stoploss = custom_stop - now = arrow.utcnow().datetime + now = dt_now() current_rate = trade.open_rate * (1 + profit) sl_flag = strategy.ft_stoploss_reached(current_rate=current_rate, trade=trade, current_time=now, current_profit=profit, @@ -506,7 +506,7 @@ def test_custom_exit(default_conf, fee, caplog) -> None: open_rate=1, ) - now = arrow.utcnow().datetime + now = dt_now() res = strategy.should_exit(trade, 1, now, enter=False, exit_=False, low=None, high=None) @@ -554,7 +554,7 @@ def test_should_sell(default_conf, fee) -> None: exchange='binance', open_rate=1, ) - now = arrow.utcnow().datetime + now = dt_now() res = strategy.should_exit(trade, 1, now, enter=False, exit_=False, low=None, high=None) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 8aa3f63d5..43204fb4d 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -22,6 +22,7 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections.iprotection import ProtectionReturn +from freqtrade.util.datetime_helpers import dt_now from freqtrade.worker import Worker from tests.conftest import (EXMS, create_mock_trades, create_mock_trades_usdt, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, @@ -2013,7 +2014,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_orde trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = '100' - trade.stoploss_last_update = arrow.utcnow().datetime + trade.stoploss_last_update = dt_now() trade.orders.append( Order( ft_order_side='stoploss', @@ -2123,7 +2124,7 @@ def test_exit_positions(mocker, default_conf_usdt, limit_order, is_short, caplog fee_open=0.001, fee_close=0.001, open_rate=0.01, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), stake_amount=0.01, amount=11, exchange="binance", @@ -2169,7 +2170,7 @@ def test_exit_positions_exception(mocker, default_conf_usdt, limit_order, caplog fee_open=0.001, fee_close=0.001, open_rate=0.01, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), stake_amount=0.01, amount=11, exchange="binance", @@ -2218,7 +2219,7 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_order, is_short, ca fee_open=0.001, fee_close=0.001, open_rate=0.01, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), amount=11, exchange="binance", is_short=is_short, @@ -2291,7 +2292,7 @@ def test_update_trade_state_withorderdict( amount=amount, exchange='binance', open_rate=2.0, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, open_order_id=order_id, @@ -2378,7 +2379,7 @@ def test_update_trade_state_sell( open_rate=0.245441, fee_open=0.0025, fee_close=0.0025, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), open_order_id=open_order['id'], is_open=True, interest_rate=0.0005, @@ -3420,7 +3421,7 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: fee_open=fee.return_value, fee_close=fee.return_value, close_rate=0.555, - close_date=arrow.utcnow().datetime, + close_date=dt_now(), exit_reason="sell_reason_whatever", stake_amount=0.245441 * 2, ) @@ -5437,7 +5438,7 @@ def test_reupdate_enter_order_fees(mocker, default_conf_usdt, fee, caplog, is_sh stake_amount=60.0, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), is_open=True, amount=30, open_rate=2.0, @@ -5573,7 +5574,7 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor fee_open=0.001, fee_close=0.001, open_rate=entry_order['price'], - open_date=arrow.utcnow().datetime, + open_date=dt_now(), stake_amount=entry_order['cost'], amount=entry_order['amount'], exchange="binance", From d131dd40500c42c65a212f3cd41bb127685c1342 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 11:12:32 +0200 Subject: [PATCH 18/56] Fix wrong transition --- freqtrade/edge/edge_positioning.py | 4 ++-- freqtrade/rpc/rpc.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index af66345e0..f2df0d3f2 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -80,8 +80,8 @@ class Edge: self._stoploss_range_step ) - self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % ( - dt_now() - timedelta(days=self._since_number_of_days)).strftime('YYYYMMDD')) + self._timerange: TimeRange = TimeRange.parse_timerange( + f"{(dt_now() - timedelta(days=self._since_number_of_days)).strftime('%Y%m%d')}-") if config.get('fee'): self.fee = config['fee'] else: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2e256ee98..245575eb7 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -32,6 +32,7 @@ from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.rpc.rpc_types import RPCSendMsg +from freqtrade.util.datetime_helpers import dt_now from freqtrade.wallets import PositionWallet, Wallet @@ -1252,7 +1253,7 @@ class RPC: df_analyzed = strategy.analyze_ticker(_data[pair], {'pair': pair}) return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe, - df_analyzed, arrow.Arrow.utcnow().datetime) + df_analyzed, dt_now()) def _rpc_plot_config(self) -> Dict[str, Any]: if (self._freqtrade.strategy.plot_config and From 29fdcdbf5698b30cd37d9ba13750750ebd8968c2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 11:19:36 +0200 Subject: [PATCH 19/56] reduce arrow in tests --- tests/conftest.py | 26 +++++++++++++------------ tests/edge/test_edge.py | 5 +++-- tests/optimize/test_optimize_reports.py | 9 +++++---- tests/test_freqtradebot.py | 2 +- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 70d15c6df..bed5c61de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,6 +23,8 @@ from freqtrade.exchange.exchange import timeframe_to_minutes from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import LocalTrade, Order, Trade, init_db from freqtrade.resolvers import ExchangeResolver +from freqtrade.util import dt_ts +from freqtrade.util.datetime_helpers import dt_now from freqtrade.worker import Worker from tests.conftest_trades import (leverage_trade, mock_trade_1, mock_trade_2, mock_trade_3, mock_trade_4, mock_trade_5, mock_trade_6, short_trade) @@ -1663,8 +1665,8 @@ def limit_buy_order_open(): 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', - 'timestamp': arrow.utcnow().int_timestamp * 1000, - 'datetime': arrow.utcnow().isoformat(), + 'timestamp': dt_ts(), + 'datetime': dt_now().isoformat(), 'price': 0.00001099, 'average': 0.00001099, 'amount': 90.99181073, @@ -1823,8 +1825,8 @@ def limit_sell_order_open(): 'type': 'limit', 'side': 'sell', 'symbol': 'mocked', - 'datetime': arrow.utcnow().isoformat(), - 'timestamp': arrow.utcnow().int_timestamp * 1000, + 'datetime': dt_now().isoformat(), + 'timestamp': dt_ts(), 'price': 0.00001173, 'amount': 90.99181073, 'filled': 0.0, @@ -2838,8 +2840,8 @@ def limit_buy_order_usdt_open(): 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', - 'datetime': arrow.utcnow().isoformat(), - 'timestamp': arrow.utcnow().int_timestamp * 1000, + 'datetime': dt_now().isoformat(), + 'timestamp': dt_ts(), 'price': 2.00, 'average': 2.00, 'amount': 30.0, @@ -2866,8 +2868,8 @@ def limit_sell_order_usdt_open(): 'type': 'limit', 'side': 'sell', 'symbol': 'mocked', - 'datetime': arrow.utcnow().isoformat(), - 'timestamp': arrow.utcnow().int_timestamp * 1000, + 'datetime': dt_now().isoformat(), + 'timestamp': dt_ts(), 'price': 2.20, 'amount': 30.0, 'cost': 66.0, @@ -2893,8 +2895,8 @@ def market_buy_order_usdt(): 'type': 'market', 'side': 'buy', 'symbol': 'mocked', - 'timestamp': arrow.utcnow().int_timestamp * 1000, - 'datetime': arrow.utcnow().isoformat(), + 'timestamp': dt_ts(), + 'datetime': dt_now().isoformat(), 'price': 2.00, 'amount': 30.0, 'filled': 30.0, @@ -2950,8 +2952,8 @@ def market_sell_order_usdt(): 'type': 'market', 'side': 'sell', 'symbol': 'mocked', - 'timestamp': arrow.utcnow().int_timestamp * 1000, - 'datetime': arrow.utcnow().isoformat(), + 'timestamp': dt_ts(), + 'datetime': dt_now().isoformat(), 'price': 2.20, 'amount': 30.0, 'filled': 30.0, diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index be0346b78..a60a785a1 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -14,6 +14,7 @@ from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.enums import ExitType from freqtrade.exceptions import OperationalException +from freqtrade.util.datetime_helpers import dt_ts from tests.conftest import EXMS, get_patched_freqtradebot, log_has from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset) @@ -220,7 +221,7 @@ def test_edge_heartbeat_calculate(mocker, edge_conf): heartbeat = edge_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached - edge._last_updated = arrow.utcnow().int_timestamp - heartbeat + 1 + edge._last_updated = dt_ts() - heartbeat + 1 assert edge.calculate(edge_conf['exchange']['pair_whitelist']) is False @@ -268,7 +269,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf): assert edge.calculate(edge_conf['exchange']['pair_whitelist']) assert len(edge._cached_pairs) == 2 - assert edge._last_updated <= arrow.utcnow().int_timestamp + 2 + assert edge._last_updated <= dt_ts() + 2 def test_edge_process_no_data(mocker, edge_conf, caplog): diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 4e3803f17..14038705f 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -25,6 +25,7 @@ from freqtrade.optimize.optimize_reports import (_get_resample_from_period, gene store_backtest_stats, text_table_bt_results, text_table_exit_reason, text_table_strategy) from freqtrade.resolvers.strategy_resolver import StrategyResolver +from freqtrade.util import dt_ts from tests.conftest import CURRENT_TEST_STRATEGY from tests.data.test_history import _clean_test_file @@ -106,8 +107,8 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): 'canceled_trade_entries': 0, 'canceled_entry_orders': 0, 'replaced_entry_orders': 0, - 'backtest_start_time': Arrow.utcnow().int_timestamp, - 'backtest_end_time': Arrow.utcnow().int_timestamp, + 'backtest_start_time': dt_ts() // 1000, + 'backtest_end_time': dt_ts() // 1000, 'run_id': '123', } } @@ -161,8 +162,8 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): 'canceled_trade_entries': 0, 'canceled_entry_orders': 0, 'replaced_entry_orders': 0, - 'backtest_start_time': Arrow.utcnow().int_timestamp, - 'backtest_end_time': Arrow.utcnow().int_timestamp, + 'backtest_start_time': dt_ts() // 1000, + 'backtest_end_time': dt_ts() // 1000, 'run_id': '124', } } diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 43204fb4d..e44f04aa0 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -6003,7 +6003,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: 'ft_is_open': False, 'id': '651', 'order_id': '651', - 'datetime': arrow.utcnow().isoformat(), + 'datetime': dt_now().isoformat(), } mocker.patch(f'{EXMS}.create_order', MagicMock(return_value=closed_dca_order_1)) From c0713eb77ffe361d8a0600046d4e8921e087674c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 11:28:40 +0200 Subject: [PATCH 20/56] More tests to dt_helpers --- tests/conftest.py | 4 ++-- tests/data/test_btanalysis.py | 33 ++++++++++++++------------- tests/persistence/test_persistence.py | 2 +- tests/test_timerange.py | 2 +- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index bed5c61de..55e6fa607 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2598,7 +2598,7 @@ def open_trade(): fee_open=0.0, fee_close=0.0, stake_amount=1, - open_date=arrow.utcnow().shift(minutes=-601).datetime, + open_date=dt_now() - timedelta(minutes=601), is_open=True ) trade.orders = [ @@ -2636,7 +2636,7 @@ def open_trade_usdt(): fee_open=0.0, fee_close=0.0, stake_amount=60.0, - open_date=arrow.utcnow().shift(minutes=-601).datetime, + open_date=dt_now() - timedelta(minutes=601), is_open=True ) trade.orders = [ diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 2c5515f7c..6550fb314 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -1,3 +1,4 @@ +from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import MagicMock @@ -162,25 +163,25 @@ def test_extract_trades_of_period(testdatadir): {'pair': [pair, pair, pair, pair], 'profit_ratio': [0.0, 0.1, -0.2, -0.5], 'profit_abs': [0.0, 1, -2, -5], - 'open_date': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime, - Arrow(2017, 11, 14, 9, 41, 0).datetime, - Arrow(2017, 11, 14, 14, 20, 0).datetime, - Arrow(2017, 11, 15, 3, 40, 0).datetime, + 'open_date': to_datetime([datetime(2017, 11, 13, 15, 40, 0, tzinfo=timezone.utc), + datetime(2017, 11, 14, 9, 41, 0, tzinfo=timezone.utc), + datetime(2017, 11, 14, 14, 20, 0, tzinfo=timezone.utc), + datetime(2017, 11, 15, 3, 40, 0, tzinfo=timezone.utc), ], utc=True ), - 'close_date': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime, - Arrow(2017, 11, 14, 10, 41, 0).datetime, - Arrow(2017, 11, 14, 15, 25, 0).datetime, - Arrow(2017, 11, 15, 3, 55, 0).datetime, + 'close_date': to_datetime([datetime(2017, 11, 13, 16, 40, 0, tzinfo=timezone.utc), + datetime(2017, 11, 14, 10, 41, 0, tzinfo=timezone.utc), + datetime(2017, 11, 14, 15, 25, 0, tzinfo=timezone.utc), + datetime(2017, 11, 15, 3, 55, 0, tzinfo=timezone.utc), ], utc=True) }) trades1 = extract_trades_of_period(data, trades) # First and last trade are dropped as they are out of range assert len(trades1) == 2 - assert trades1.iloc[0].open_date == Arrow(2017, 11, 14, 9, 41, 0).datetime - assert trades1.iloc[0].close_date == Arrow(2017, 11, 14, 10, 41, 0).datetime - assert trades1.iloc[-1].open_date == Arrow(2017, 11, 14, 14, 20, 0).datetime - assert trades1.iloc[-1].close_date == Arrow(2017, 11, 14, 15, 25, 0).datetime + assert trades1.iloc[0].open_date == datetime(2017, 11, 14, 9, 41, 0, tzinfo=timezone.utc) + assert trades1.iloc[0].close_date == datetime(2017, 11, 14, 10, 41, 0, tzinfo=timezone.utc) + assert trades1.iloc[-1].open_date == datetime(2017, 11, 14, 14, 20, 0, tzinfo=timezone.utc) + assert trades1.iloc[-1].close_date == datetime(2017, 11, 14, 15, 25, 0, tzinfo=timezone.utc) def test_analyze_trade_parallelism(testdatadir): @@ -454,8 +455,8 @@ def test_calculate_max_drawdown_abs(profits, relative, highd, lowd, result, resu [1000, 500, 1000, 11000, 10000] # absolute results [1000, 50%, 0%, 0%, ~9%] # Relative drawdowns """ - init_date = Arrow(2020, 1, 1) - dates = [init_date.shift(days=i) for i in range(len(profits))] + init_date = datetime(2020, 1, 1, tzinfo=timezone.utc) + dates = [init_date + timedelta(days=i) for i in range(len(profits))] df = DataFrame(zip(profits, dates), columns=['profit_abs', 'open_date']) # sort by profit and reset index df = df.sort_values('profit_abs').reset_index(drop=True) @@ -467,8 +468,8 @@ def test_calculate_max_drawdown_abs(profits, relative, highd, lowd, result, resu assert isinstance(drawdown, float) assert isinstance(drawdown_rel, float) - assert hdate == init_date.shift(days=highd) - assert ldate == init_date.shift(days=lowd) + assert hdate == init_date + timedelta(days=highd) + assert ldate == init_date + timedelta(days=lowd) # High must be before low assert hdate < ldate diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 92a0592cb..cb6066ab5 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -636,7 +636,7 @@ def test_trade_close(fee): assert pytest.approx(trade.close_profit) == 0.094513715 assert trade.close_date is not None - new_date = arrow.Arrow(2020, 2, 2, 15, 6, 1).datetime, + new_date = datetime(2020, 2, 2, 15, 6, 1), assert trade.close_date != new_date # Close should NOT update close_date if the trade has been closed already assert trade.is_open is False diff --git a/tests/test_timerange.py b/tests/test_timerange.py index 993b24d95..8247b60be 100644 --- a/tests/test_timerange.py +++ b/tests/test_timerange.py @@ -69,7 +69,7 @@ def test_subtract_start(): def test_adjust_start_if_necessary(): - min_date = arrow.Arrow(2017, 11, 14, 21, 15, 00) + min_date = datetime(2017, 11, 14, 21, 15, 00, tzinfo=timezone.utc) x = TimeRange('date', 'date', 1510694100, 1510780500) # Adjust by 20 candles - min_date == startts From 915cb5ffbdd84602356a3291d0a6dc4bcaf70125 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 17:36:53 +0200 Subject: [PATCH 21/56] add dt_utc helper --- freqtrade/util/__init__.py | 5 +++-- freqtrade/util/datetime_helpers.py | 6 ++++++ tests/utils/test_datetime_helpers.py | 9 ++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index b28544fd3..aa0ad05f2 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,13 +1,14 @@ -from freqtrade.util.datetime_helpers import dt_floor_day, dt_from_ts, dt_now, dt_ts +from freqtrade.util.datetime_helpers import dt_floor_day, dt_from_ts, dt_now, dt_ts, dt_utc from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache __all__ = [ + 'dt_floor_day', 'dt_from_ts', 'dt_now', 'dt_ts', - 'dt_floor_day', + 'dt_utc', 'FtPrecise', 'PeriodicCache', ] diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index 228bca185..c44761fd4 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -7,6 +7,12 @@ def dt_now() -> datetime: return datetime.now(timezone.utc) +def dt_utc(year: int, month: int, day: int, hour: int = 0, minute: int = 0, second: int = 0, + microsecond: int = 0) -> datetime: + """Return a datetime in UTC.""" + return datetime(year, month, day, hour, minute, second, microsecond, tzinfo=timezone.utc) + + def dt_ts(dt: Optional[datetime] = None) -> int: """ Return dt in ms as a timestamp in UTC. diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index 7cff44847..f1c445caf 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta, timezone import pytest import time_machine -from freqtrade.util import dt_floor_day, dt_from_ts, dt_now, dt_ts +from freqtrade.util import dt_floor_day, dt_from_ts, dt_now, dt_ts, dt_utc def test_dt_now(): @@ -21,6 +21,13 @@ def test_dt_now(): assert dt_ts(now) == int(now.timestamp() * 1000) +def test_dt_utc(): + assert dt_utc(2023, 5, 5) == datetime(2023, 5, 5, tzinfo=timezone.utc) + assert dt_utc(2023, 5, 5, 0, 0, 0, 555500) == datetime(2023, 5, 5, 0, 0, 0, 555500, + tzinfo=timezone.utc) + + + @pytest.mark.parametrize('as_ms', [True, False]) def test_dt_from_ts(as_ms): multi = 1000 if as_ms else 1 From 7a2ff6025587b0d92d78981db0e9173c7f8a0e23 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 17:46:56 +0200 Subject: [PATCH 22/56] Fix more tests --- tests/optimize/__init__.py | 7 ++-- tests/optimize/test_backtesting.py | 9 +++-- .../test_backtesting_adjust_position.py | 10 ++--- tests/optimize/test_hyperopt.py | 33 ++++++++-------- tests/optimize/test_optimize_reports.py | 38 +++++++++---------- tests/test_freqtradebot.py | 8 ++-- 6 files changed, 54 insertions(+), 51 deletions(-) diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index a3dd59004..b95764ba5 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -1,13 +1,14 @@ +from datetime import timedelta from typing import Dict, List, NamedTuple, Optional -import arrow from pandas import DataFrame from freqtrade.enums import ExitType from freqtrade.exchange import timeframe_to_minutes +from freqtrade.util.datetime_helpers import dt_utc -tests_start_time = arrow.get(2018, 10, 3) +tests_start_time = dt_utc(2018, 10, 3) tests_timeframe = '1h' @@ -46,7 +47,7 @@ class BTContainer(NamedTuple): def _get_frame_time_from_offset(offset): minutes = offset * timeframe_to_minutes(tests_timeframe) - return tests_start_time.shift(minutes=minutes).datetime + return tests_start_time + timedelta(minutes=minutes) def _build_backtest_dataframe(data): diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index a9e87347c..0af75111a 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -26,6 +26,7 @@ from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade, Trade from freqtrade.resolvers import StrategyResolver +from freqtrade.util.datetime_helpers import dt_utc from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -710,11 +711,11 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'stake_amount': [0.001, 0.001], 'max_stake_amount': [0.001, 0.001], 'amount': [0.00957442, 0.0097064], - 'open_date': pd.to_datetime([Arrow(2018, 1, 29, 18, 40, 0).datetime, - Arrow(2018, 1, 30, 3, 30, 0).datetime], utc=True + 'open_date': pd.to_datetime([dt_utc(2018, 1, 29, 18, 40, 0), + dt_utc(2018, 1, 30, 3, 30, 0)], utc=True ), - 'close_date': pd.to_datetime([Arrow(2018, 1, 29, 22, 35, 0).datetime, - Arrow(2018, 1, 30, 4, 10, 0).datetime], utc=True), + 'close_date': pd.to_datetime([dt_utc(2018, 1, 29, 22, 35, 0), + dt_utc(2018, 1, 30, 4, 10, 0)], utc=True), 'open_rate': [0.104445, 0.10302485], 'close_rate': [0.104969, 0.103541], 'fee_open': [0.0025, 0.0025], diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 0d57ff89a..ce26e836e 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -5,13 +5,13 @@ from unittest.mock import MagicMock import pandas as pd import pytest -from arrow import Arrow from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.history import get_timerange from freqtrade.enums import ExitType, TradingMode from freqtrade.optimize.backtesting import Backtesting +from freqtrade.util.datetime_helpers import dt_utc from tests.conftest import EXMS, patch_exchange @@ -52,11 +52,11 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> 'stake_amount': [500.0, 100.0], 'max_stake_amount': [500.0, 100], 'amount': [4806.87657523, 970.63960782], - 'open_date': pd.to_datetime([Arrow(2018, 1, 29, 18, 40, 0).datetime, - Arrow(2018, 1, 30, 3, 30, 0).datetime], utc=True + 'open_date': pd.to_datetime([dt_utc(2018, 1, 29, 18, 40, 0), + dt_utc(2018, 1, 30, 3, 30, 0)], utc=True ), - 'close_date': pd.to_datetime([Arrow(2018, 1, 29, 22, 00, 0).datetime, - Arrow(2018, 1, 30, 4, 10, 0).datetime], utc=True), + 'close_date': pd.to_datetime([dt_utc(2018, 1, 29, 22, 00, 0), + dt_utc(2018, 1, 30, 4, 10, 0)], utc=True), 'open_rate': [0.10401764894444211, 0.10302485], 'close_rate': [0.10453904066847439, 0.103541], 'fee_open': [0.0025, 0.0025], diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 786720030..63691b08d 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -20,6 +20,7 @@ from freqtrade.optimize.hyperopt_tools import HyperoptTools from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.optimize.space import SKDecimal from freqtrade.strategy import IntParameter +from freqtrade.util import dt_utc from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, get_args, get_markets, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -349,14 +350,14 @@ def test_hyperopt_format_results(hyperopt): "UNITTEST/BTC", "UNITTEST/BTC"], "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780], "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], - "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_date": [dt_utc(2017, 11, 14, 19, 32, 00), + dt_utc(2017, 11, 14, 21, 36, 00), + dt_utc(2017, 11, 14, 22, 12, 00), + dt_utc(2017, 11, 14, 22, 44, 00)], + "close_date": [dt_utc(2017, 11, 14, 21, 35, 00), + dt_utc(2017, 11, 14, 22, 10, 00), + dt_utc(2017, 11, 14, 22, 43, 00), + dt_utc(2017, 11, 14, 22, 58, 00)], "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "trade_duration": [123, 34, 31, 14], @@ -423,14 +424,14 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: "UNITTEST/BTC", "UNITTEST/BTC"], "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780], "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], - "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_date": [dt_utc(2017, 11, 14, 19, 32, 00), + dt_utc(2017, 11, 14, 21, 36, 00), + dt_utc(2017, 11, 14, 22, 12, 00), + dt_utc(2017, 11, 14, 22, 44, 00)], + "close_date": [dt_utc(2017, 11, 14, 21, 35, 00), + dt_utc(2017, 11, 14, 22, 10, 00), + dt_utc(2017, 11, 14, 22, 43, 00), + dt_utc(2017, 11, 14, 22, 58, 00)], "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "trade_duration": [123, 34, 31, 14], diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 14038705f..82e8a46fb 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -6,7 +6,6 @@ from shutil import copyfile import joblib import pandas as pd import pytest -from arrow import Arrow from freqtrade.configuration import TimeRange from freqtrade.constants import BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN @@ -26,6 +25,7 @@ from freqtrade.optimize.optimize_reports import (_get_resample_from_period, gene text_table_exit_reason, text_table_strategy) from freqtrade.resolvers.strategy_resolver import StrategyResolver from freqtrade.util import dt_ts +from freqtrade.util.datetime_helpers import dt_from_ts, dt_utc from tests.conftest import CURRENT_TEST_STRATEGY from tests.data.test_history import _clean_test_file @@ -81,14 +81,14 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): "UNITTEST/BTC", "UNITTEST/BTC"], "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780], "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], - "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_date": [dt_utc(2017, 11, 14, 19, 32, 00), + dt_utc(2017, 11, 14, 21, 36, 00), + dt_utc(2017, 11, 14, 22, 12, 00), + dt_utc(2017, 11, 14, 22, 44, 00)], + "close_date": [dt_utc(2017, 11, 14, 21, 35, 00), + dt_utc(2017, 11, 14, 22, 10, 00), + dt_utc(2017, 11, 14, 22, 43, 00), + dt_utc(2017, 11, 14, 22, 58, 00)], "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "trade_duration": [123, 34, 31, 14], @@ -113,8 +113,8 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): } } timerange = TimeRange.parse_timerange('1510688220-1510700340') - min_date = Arrow.fromtimestamp(1510688220) - max_date = Arrow.fromtimestamp(1510700340) + min_date = dt_from_ts(1510688220) + max_date = dt_from_ts(1510700340) btdata = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, fill_up_missing=True) @@ -136,14 +136,14 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): {"pair": ["UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC"], "profit_ratio": [0.003312, 0.010801, -0.013803, 0.002780], "profit_abs": [0.000003, 0.000011, -0.000014, 0.000003], - "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_date": [dt_utc(2017, 11, 14, 19, 32, 00), + dt_utc(2017, 11, 14, 21, 36, 00), + dt_utc(2017, 11, 14, 22, 12, 00), + dt_utc(2017, 11, 14, 22, 44, 00)], + "close_date": [dt_utc(2017, 11, 14, 21, 35, 00), + dt_utc(2017, 11, 14, 22, 10, 00), + dt_utc(2017, 11, 14, 22, 43, 00), + dt_utc(2017, 11, 14, 22, 58, 00)], "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.0032903, 0.003217], "trade_duration": [123, 34, 31, 14], diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e44f04aa0..288ad58f0 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -22,7 +22,7 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections.iprotection import ProtectionReturn -from freqtrade.util.datetime_helpers import dt_now +from freqtrade.util.datetime_helpers import dt_now, dt_utc from freqtrade.worker import Worker from tests.conftest import (EXMS, create_mock_trades, create_mock_trades_usdt, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, @@ -5712,9 +5712,9 @@ def test_update_funding_fees( default_conf['trading_mode'] = 'futures' default_conf['margin_mode'] = 'isolated' - date_midnight = arrow.get('2021-09-01 00:00:00').datetime - date_eight = arrow.get('2021-09-01 08:00:00').datetime - date_sixteen = arrow.get('2021-09-01 16:00:00').datetime + date_midnight = dt_utc(2021, 9, 1) + date_eight = dt_utc(2021, 9, 1, 8) + date_sixteen = dt_utc(2021, 9, 1, 16) columns = ['date', 'open', 'high', 'low', 'close', 'volume'] # 16:00 entry is actually never used # But should be kept in the test to ensure we're filtering correctly. From 3a4d103bc84e7ed0149f7aad96b14f7a236a83ef Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 17:49:43 +0200 Subject: [PATCH 23/56] Properly check wallets with new type --- tests/test_wallets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_wallets.py b/tests/test_wallets.py index 09adf6e15..c3ff4ccd0 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -45,7 +45,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets._wallets['GAS'].total == 0.260739 assert freqtrade.wallets.get_free('BNT') == 1.0 assert 'USDT' in freqtrade.wallets._wallets - assert freqtrade.wallets._last_wallet_refresh > 0 + assert freqtrade.wallets._last_wallet_refresh is not None mocker.patch.multiple( EXMS, get_balances=MagicMock(return_value={ @@ -332,7 +332,7 @@ def test_sync_wallet_futures_live(mocker, default_conf): assert 'USDT' in freqtrade.wallets._wallets assert 'ETH/USDT:USDT' in freqtrade.wallets._positions - assert freqtrade.wallets._last_wallet_refresh > 0 + assert freqtrade.wallets._last_wallet_refresh is not None # Remove ETH/USDT:USDT position del mock_result[0] From 9421ca2628dfe097d0c1981e2f819d1369de7030 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 17:55:24 +0200 Subject: [PATCH 24/56] Remove arrow from test_persistence --- tests/persistence/test_persistence.py | 47 ++++++++++++----------- tests/strategy/test_interface.py | 54 +++++++++++++-------------- 2 files changed, 50 insertions(+), 51 deletions(-) diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index cb6066ab5..4aa3b1e96 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -2,7 +2,6 @@ from datetime import datetime, timedelta, timezone from types import FunctionType -import arrow import pytest from sqlalchemy import select @@ -10,7 +9,7 @@ from freqtrade.constants import CUSTOM_TAG_MAX_LENGTH, DATETIME_PRINT_FORMAT from freqtrade.enums import TradingMode from freqtrade.exceptions import DependencyException from freqtrade.persistence import LocalTrade, Order, Trade, init_db -from freqtrade.util.datetime_helpers import dt_now +from freqtrade.util import dt_now from tests.conftest import create_mock_trades, create_mock_trades_with_leverage, log_has, log_has_re @@ -1327,7 +1326,7 @@ def test_to_json(fee): amount_requested=123.0, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), open_rate=0.123, exchange='binance', enter_tag=None, @@ -1412,8 +1411,8 @@ def test_to_json(fee): amount_requested=101.0, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, - close_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=2), + close_date=dt_now() - timedelta(hours=1), open_rate=0.123, close_rate=0.125, enter_tag='buys_signal_001', @@ -1497,7 +1496,7 @@ def test_stoploss_reinitialization(default_conf, fee): pair='ADA/USDT', stake_amount=30.0, fee_open=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=30.0, fee_close=fee.return_value, exchange='binance', @@ -1558,7 +1557,7 @@ def test_stoploss_reinitialization_leverage(default_conf, fee): pair='ADA/USDT', stake_amount=30.0, fee_open=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=30.0, fee_close=fee.return_value, exchange='binance', @@ -1620,7 +1619,7 @@ def test_stoploss_reinitialization_short(default_conf, fee): pair='ADA/USDT', stake_amount=0.001, fee_open=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=10, fee_close=fee.return_value, exchange='binance', @@ -1679,7 +1678,7 @@ def test_update_fee(fee): pair='ADA/USDT', stake_amount=30.0, fee_open=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=30.0, fee_close=fee.return_value, exchange='binance', @@ -1718,7 +1717,7 @@ def test_fee_updated(fee): pair='ADA/USDT', stake_amount=30.0, fee_open=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=30.0, fee_close=fee.return_value, exchange='binance', @@ -2093,7 +2092,7 @@ def test_recalc_trade_from_orders(fee): trade = Trade( pair='ADA/USDT', stake_amount=o1_cost, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=o1_amount, fee_open=fee.return_value, fee_close=fee.return_value, @@ -2168,8 +2167,8 @@ def test_recalc_trade_from_orders(fee): filled=o2_amount, remaining=0, cost=o2_cost, - order_date=arrow.utcnow().shift(hours=-1).datetime, - order_filled_date=arrow.utcnow().shift(hours=-1).datetime, + order_date=dt_now() - timedelta(hours=1), + order_filled_date=dt_now() - timedelta(hours=1), ) trade.orders.append(order2) trade.recalc_trade_from_orders() @@ -2202,8 +2201,8 @@ def test_recalc_trade_from_orders(fee): filled=o3_amount, remaining=0, cost=o3_cost, - order_date=arrow.utcnow().shift(hours=-1).datetime, - order_filled_date=arrow.utcnow().shift(hours=-1).datetime, + order_date=dt_now() - timedelta(hours=1), + order_filled_date=dt_now() - timedelta(hours=1), ) trade.orders.append(order3) trade.recalc_trade_from_orders() @@ -2258,7 +2257,7 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): trade = Trade( pair='ADA/USDT', stake_amount=o1_cost, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=o1_amount, fee_open=fee.return_value, fee_close=fee.return_value, @@ -2310,8 +2309,8 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): filled=o1_amount, remaining=0, cost=o1_cost, - order_date=arrow.utcnow().shift(hours=-1).datetime, - order_filled_date=arrow.utcnow().shift(hours=-1).datetime, + order_date=dt_now() - timedelta(hours=1), + order_filled_date=dt_now() - timedelta(hours=1), ) trade.orders.append(order2) trade.recalc_trade_from_orders() @@ -2338,8 +2337,8 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): filled=0, remaining=4, cost=5, - order_date=arrow.utcnow().shift(hours=-1).datetime, - order_filled_date=arrow.utcnow().shift(hours=-1).datetime, + order_date=dt_now() - timedelta(hours=1), + order_filled_date=dt_now() - timedelta(hours=1), ) trade.orders.append(order3) trade.recalc_trade_from_orders() @@ -2365,8 +2364,8 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): filled=o1_amount, remaining=0, cost=o1_cost, - order_date=arrow.utcnow().shift(hours=-1).datetime, - order_filled_date=arrow.utcnow().shift(hours=-1).datetime, + order_date=dt_now() - timedelta(hours=1), + order_filled_date=dt_now() - timedelta(hours=1), ) trade.orders.append(order4) trade.recalc_trade_from_orders() @@ -2623,8 +2622,8 @@ def test_recalc_trade_from_orders_dca(data) -> None: filled=amount, remaining=0, cost=amount * price, - order_date=arrow.utcnow().shift(hours=-10 + idx).datetime, - order_filled_date=arrow.utcnow().shift(hours=-10 + idx).datetime, + order_date=dt_now() - timedelta(hours=10 + idx), + order_filled_date=dt_now() - timedelta(hours=10 + idx), ) trade.orders.append(order_obj) trade.recalc_trade_from_orders() diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index a7dde20d6..23d5e6ea3 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -331,14 +331,14 @@ def test_min_roi_reached(default_conf, fee) -> None: open_rate=1, ) - assert not strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-56).datetime) - assert strategy.min_roi_reached(trade, 0.12, arrow.utcnow().shift(minutes=-56).datetime) + assert not strategy.min_roi_reached(trade, 0.02, dt_now() - timedelta(minutes=56)) + assert strategy.min_roi_reached(trade, 0.12, dt_now() - timedelta(minutes=56)) - assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-39).datetime) - assert strategy.min_roi_reached(trade, 0.06, arrow.utcnow().shift(minutes=-39).datetime) + assert not strategy.min_roi_reached(trade, 0.04, dt_now() - timedelta(minutes=39)) + assert strategy.min_roi_reached(trade, 0.06, dt_now() - timedelta(minutes=39)) - assert not strategy.min_roi_reached(trade, -0.01, arrow.utcnow().shift(minutes=-1).datetime) - assert strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-1).datetime) + assert not strategy.min_roi_reached(trade, -0.01, dt_now() - timedelta(minutes=1)) + assert strategy.min_roi_reached(trade, 0.02, dt_now() - timedelta(minutes=1)) def test_min_roi_reached2(default_conf, fee) -> None: @@ -362,25 +362,25 @@ def test_min_roi_reached2(default_conf, fee) -> None: pair='ETH/BTC', stake_amount=0.001, amount=5, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', open_rate=1, ) - assert not strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-56).datetime) - assert strategy.min_roi_reached(trade, 0.12, arrow.utcnow().shift(minutes=-56).datetime) + assert not strategy.min_roi_reached(trade, 0.02, dt_now() - timedelta(minutes=56)) + assert strategy.min_roi_reached(trade, 0.12, dt_now() - timedelta(minutes=56)) - assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-39).datetime) - assert strategy.min_roi_reached(trade, 0.071, arrow.utcnow().shift(minutes=-39).datetime) + assert not strategy.min_roi_reached(trade, 0.04, dt_now() - timedelta(minutes=39)) + assert strategy.min_roi_reached(trade, 0.071, dt_now() - timedelta(minutes=39)) - assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-26).datetime) - assert strategy.min_roi_reached(trade, 0.06, arrow.utcnow().shift(minutes=-26).datetime) + assert not strategy.min_roi_reached(trade, 0.04, dt_now() - timedelta(minutes=26)) + assert strategy.min_roi_reached(trade, 0.06, dt_now() - timedelta(minutes=26)) # Should not trigger with 20% profit since after 55 minutes only 30% is active. - assert not strategy.min_roi_reached(trade, 0.20, arrow.utcnow().shift(minutes=-2).datetime) - assert strategy.min_roi_reached(trade, 0.31, arrow.utcnow().shift(minutes=-2).datetime) + assert not strategy.min_roi_reached(trade, 0.20, dt_now() - timedelta(minutes=2)) + assert strategy.min_roi_reached(trade, 0.31, dt_now() - timedelta(minutes=2)) def test_min_roi_reached3(default_conf, fee) -> None: @@ -396,25 +396,25 @@ def test_min_roi_reached3(default_conf, fee) -> None: pair='ETH/BTC', stake_amount=0.001, amount=5, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', open_rate=1, ) - assert not strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-56).datetime) - assert not strategy.min_roi_reached(trade, 0.12, arrow.utcnow().shift(minutes=-56).datetime) + assert not strategy.min_roi_reached(trade, 0.02, dt_now() - timedelta(minutes=56)) + assert not strategy.min_roi_reached(trade, 0.12, dt_now() - timedelta(minutes=56)) - assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-39).datetime) - assert strategy.min_roi_reached(trade, 0.071, arrow.utcnow().shift(minutes=-39).datetime) + assert not strategy.min_roi_reached(trade, 0.04, dt_now() - timedelta(minutes=39)) + assert strategy.min_roi_reached(trade, 0.071, dt_now() - timedelta(minutes=39)) - assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-26).datetime) - assert strategy.min_roi_reached(trade, 0.06, arrow.utcnow().shift(minutes=-26).datetime) + assert not strategy.min_roi_reached(trade, 0.04, dt_now() - timedelta(minutes=26)) + assert strategy.min_roi_reached(trade, 0.06, dt_now() - timedelta(minutes=26)) # Should not trigger with 20% profit since after 55 minutes only 30% is active. - assert not strategy.min_roi_reached(trade, 0.20, arrow.utcnow().shift(minutes=-2).datetime) - assert strategy.min_roi_reached(trade, 0.31, arrow.utcnow().shift(minutes=-2).datetime) + assert not strategy.min_roi_reached(trade, 0.20, dt_now() - timedelta(minutes=2)) + assert strategy.min_roi_reached(trade, 0.31, dt_now() - timedelta(minutes=2)) @pytest.mark.parametrize( @@ -450,7 +450,7 @@ def test_ft_stoploss_reached(default_conf, fee, profit, adjusted, expected, liq, pair='ETH/BTC', stake_amount=0.01, amount=1, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -499,7 +499,7 @@ def test_custom_exit(default_conf, fee, caplog) -> None: pair='ETH/BTC', stake_amount=0.01, amount=1, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -548,7 +548,7 @@ def test_should_sell(default_conf, fee) -> None: pair='ETH/BTC', stake_amount=0.01, amount=1, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', From 3ec55885bd47c4ebcafd3a673cf99f4d293e2b29 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 18:14:35 +0200 Subject: [PATCH 25/56] Remove arrow from more tests --- tests/plugins/test_pairlocks.py | 24 ++++++++++++------------ tests/rpc/test_rpc_telegram.py | 27 ++++++++++++++------------- tests/strategy/test_interface.py | 4 ++-- tests/test_freqtradebot.py | 32 ++++++++++++++++---------------- 4 files changed, 44 insertions(+), 43 deletions(-) diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index 6b7112f98..6e209df60 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -1,10 +1,10 @@ from datetime import datetime, timedelta, timezone -import arrow import pytest from freqtrade.persistence import PairLocks from freqtrade.persistence.models import PairLock +from freqtrade.util import dt_now @pytest.mark.parametrize('use_db', (False, True)) @@ -20,20 +20,20 @@ def test_PairLocks(use_db): pair = 'ETH/BTC' assert not PairLocks.is_pair_locked(pair) - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=4)) # ETH/BTC locked for 4 minutes (on both sides) assert PairLocks.is_pair_locked(pair) assert PairLocks.is_pair_locked(pair, side='long') assert PairLocks.is_pair_locked(pair, side='short') pair = 'BNB/BTC' - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime, side='long') + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=4), side='long') assert not PairLocks.is_pair_locked(pair) assert PairLocks.is_pair_locked(pair, side='long') assert not PairLocks.is_pair_locked(pair, side='short') pair = 'BNB/USDT' - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime, side='short') + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=4), side='short') assert not PairLocks.is_pair_locked(pair) assert not PairLocks.is_pair_locked(pair, side='long') assert PairLocks.is_pair_locked(pair, side='short') @@ -44,7 +44,7 @@ def test_PairLocks(use_db): # Unlocking a pair that's not locked should not raise an error PairLocks.unlock_pair(pair) - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=4)) assert PairLocks.is_pair_locked(pair) # Get both locks from above @@ -113,20 +113,20 @@ def test_PairLocks_getlongestlock(use_db): pair = 'ETH/BTC' assert not PairLocks.is_pair_locked(pair) - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=4)) # ETH/BTC locked for 4 minutes assert PairLocks.is_pair_locked(pair) lock = PairLocks.get_pair_longest_lock(pair) - assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=3) - assert lock.lock_end_time.replace(tzinfo=timezone.utc) < arrow.utcnow().shift(minutes=14) + assert lock.lock_end_time.replace(tzinfo=timezone.utc) > dt_now() + timedelta(minutes=3) + assert lock.lock_end_time.replace(tzinfo=timezone.utc) < dt_now() + timedelta(minutes=14) - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=15).datetime) + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=15)) assert PairLocks.is_pair_locked(pair) lock = PairLocks.get_pair_longest_lock(pair) # Must be longer than above - assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=14) + assert lock.lock_end_time.replace(tzinfo=timezone.utc) > dt_now() + timedelta(minutes=14) PairLocks.reset_locks() PairLocks.use_db = True @@ -143,8 +143,8 @@ def test_PairLocks_reason(use_db): assert PairLocks.use_db == use_db - PairLocks.lock_pair('XRP/USDT', arrow.utcnow().shift(minutes=4).datetime, 'TestLock1') - PairLocks.lock_pair('ETH/USDT', arrow.utcnow().shift(minutes=4).datetime, 'TestLock2') + PairLocks.lock_pair('XRP/USDT', dt_now() + timedelta(minutes=4), 'TestLock1') + PairLocks.lock_pair('ETH/USDT', dt_now() + timedelta(minutes=4), 'TestLock2') assert PairLocks.is_pair_locked('XRP/USDT') assert PairLocks.is_pair_locked('ETH/USDT') diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 0d8c98d29..96d56d13f 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -33,6 +33,7 @@ from freqtrade.persistence.models import Order from freqtrade.rpc import RPC from freqtrade.rpc.rpc import RPCException from freqtrade.rpc.telegram import Telegram, authorized_only +from freqtrade.util.datetime_helpers import dt_now from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, create_mock_trades, create_mock_trades_usdt, get_patched_freqtradebot, log_has, log_has_re, patch_exchange, patch_get_signal, patch_whitelist) @@ -1518,8 +1519,8 @@ async def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) - msg_mock.reset_mock() - PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason') - PairLocks.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef') + PairLocks.lock_pair('ETH/BTC', dt_now() + timedelta(minutes=4), 'randreason') + PairLocks.lock_pair('XRP/BTC', dt_now() + timedelta(minutes=20), 'deadbeef') await telegram._locks(update=update, context=MagicMock()) @@ -1898,7 +1899,7 @@ def test_send_msg_enter_notification(default_conf, mocker, caplog, message_type, 'current_rate': 1.099e-05, 'amount': 1333.3333333333335, 'analyzed_candle': {'open': 1.1, 'high': 2.2, 'low': 1.0, 'close': 1.5}, - 'open_date': arrow.utcnow().shift(hours=-1) + 'open_date': dt_now() + timedelta(hours=-1) } telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) @@ -1959,7 +1960,7 @@ def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) time_machine.move_to("2021-09-01 05:00:00 +00:00") - lock = PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=6).datetime, 'randreason') + lock = PairLocks.lock_pair('ETH/BTC', dt_now() + timedelta(minutes=6), 'randreason') msg = { 'type': RPCMessageType.PROTECTION_TRIGGER, } @@ -1974,7 +1975,7 @@ def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> msg = { 'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, } - lock = PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=100).datetime, 'randreason') + lock = PairLocks.lock_pair('*', dt_now() + timedelta(minutes=100), 'randreason') msg.update(lock.to_json()) telegram.send_msg(msg) assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " @@ -2005,7 +2006,7 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en 'fiat_currency': 'USD', 'open_rate': 1.099e-05, 'amount': 1333.3333333333335, - 'open_date': arrow.utcnow().shift(hours=-1) + 'open_date': dt_now() - timedelta(hours=1) }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( @@ -2032,7 +2033,7 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en 'fiat_currency': 'USD', 'open_rate': 1.099e-05, 'amount': 1333.3333333333335, - 'open_date': arrow.utcnow().shift(hours=-1) + 'open_date': dt_now() - timedelta(hours=1) }) assert msg_mock.call_args[0][0] == ( @@ -2071,7 +2072,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'fiat_currency': 'USD', 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(hours=-1), + 'open_date': dt_now() - timedelta(hours=1), 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ( @@ -2107,7 +2108,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'fiat_currency': 'USD', 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), + 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), 'close_date': arrow.utcnow(), 'stake_amount': 0.01, 'sub_trade': True, @@ -2144,7 +2145,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'stake_currency': 'ETH', 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), + 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ( @@ -2226,7 +2227,7 @@ def test_send_msg_sell_fill_notification(default_conf, mocker, direction, 'stake_currency': 'ETH', 'enter_tag': enter_signal, 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), + 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), 'close_date': arrow.utcnow(), }) @@ -2317,7 +2318,7 @@ def test_send_msg_buy_notification_no_fiat( 'fiat_currency': None, 'current_rate': 1.099e-05, 'amount': 1333.3333333333335, - 'open_date': arrow.utcnow().shift(hours=-1) + 'open_date': dt_now() - timedelta(hours=1) }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' @@ -2363,7 +2364,7 @@ def test_send_msg_sell_notification_no_fiat( 'fiat_currency': 'USD', 'enter_tag': enter_signal, 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(hours=-2, minutes=-35, seconds=-3), + 'open_date': dt_now() - timedelta(hours=2, minutes=35, seconds=3), 'close_date': arrow.utcnow(), }) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 23d5e6ea3..8be0986cd 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -729,7 +729,7 @@ def test_is_pair_locked(default_conf): pair = 'ETH/BTC' assert not strategy.is_pair_locked(pair) - strategy.lock_pair(pair, arrow.now(timezone.utc).shift(minutes=4).datetime) + strategy.lock_pair(pair, dt_now() + timedelta(minutes=4)) # ETH/BTC locked for 4 minutes assert strategy.is_pair_locked(pair) @@ -747,7 +747,7 @@ def test_is_pair_locked(default_conf): # Lock with reason reason = "TestLockR" - strategy.lock_pair(pair, arrow.now(timezone.utc).shift(minutes=4).datetime, reason) + strategy.lock_pair(pair, dt_now() + timedelta(minutes=4), reason) assert strategy.is_pair_locked(pair) strategy.unlock_reason(reason) assert not strategy.is_pair_locked(pair) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 288ad58f0..9a2e5bd88 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4,10 +4,10 @@ import logging import time from copy import deepcopy +from datetime import timedelta from typing import List from unittest.mock import ANY, MagicMock, PropertyMock, patch -import arrow import pytest from pandas import DataFrame from sqlalchemy import select @@ -446,7 +446,7 @@ def test_enter_positions_global_pairlock(default_conf_usdt, ticker_usdt, limit_b assert not log_has_re(message, caplog) caplog.clear() - PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because', side='*') + PairLocks.lock_pair('*', dt_now() + timedelta(minutes=20), 'Just because', side='*') n = freqtrade.enter_positions() assert n == 0 assert log_has_re(message, caplog) @@ -467,7 +467,7 @@ def test_handle_protections(mocker, default_conf_usdt, fee, is_short): freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) freqtrade.protections._protection_handlers[1].global_stop = MagicMock( - return_value=ProtectionReturn(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) + return_value=ProtectionReturn(True, dt_now() + timedelta(hours=1), "asdf")) create_mock_trades(fee, is_short) freqtrade.handle_protections('ETC/BTC', '*') send_msg_mock = freqtrade.rpc.send_msg @@ -1263,7 +1263,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ }]) trade.stoploss_order_id = "107" trade.is_open = True - trade.stoploss_last_update = arrow.utcnow().shift(hours=-1).datetime + trade.stoploss_last_update = dt_now() - timedelta(hours=1) trade.stop_loss = 24 trade.exit_reason = None trade.orders.append( @@ -1412,7 +1412,7 @@ def test_handle_stoploss_on_exchange_partial_cancel_here( }) mocker.patch(f'{EXMS}.fetch_stoploss_order', stoploss_order_hit) mocker.patch(f'{EXMS}.cancel_stoploss_order_with_result', stoploss_order_cancel) - trade.stoploss_last_update = arrow.utcnow().shift(minutes=-10).datetime + trade.stoploss_last_update = dt_now() - timedelta(minutes=10) assert freqtrade.handle_stoploss_on_exchange(trade) is False # Canceled Stoploss filled partially ... @@ -1632,7 +1632,7 @@ def test_handle_stoploss_on_exchange_trailing( trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = '100' - trade.stoploss_last_update = arrow.utcnow().shift(minutes=-20).datetime + trade.stoploss_last_update = dt_now() - timedelta(minutes=20) trade.orders.append( Order( ft_order_side='stoploss', @@ -1763,7 +1763,7 @@ def test_handle_stoploss_on_exchange_trailing_error( trade.open_order_id = None trade.stoploss_order_id = "abcd" trade.stop_loss = 0.2 - trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime.replace(tzinfo=None) + trade.stoploss_last_update = (dt_now() - timedelta(minutes=601)).replace(tzinfo=None) trade.is_short = is_short stoploss_order_hanging = { @@ -1787,7 +1787,7 @@ def test_handle_stoploss_on_exchange_trailing_error( assert stoploss.call_count == 1 # Fail creating stoploss order - trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime + trade.stoploss_last_update = dt_now() - timedelta(minutes=601) caplog.clear() cancel_mock = mocker.patch(f'{EXMS}.cancel_stoploss_order') mocker.patch(f'{EXMS}.create_stoploss', side_effect=ExchangeError()) @@ -1876,7 +1876,7 @@ def test_handle_stoploss_on_exchange_custom_stop( trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = '100' - trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime + trade.stoploss_last_update = dt_now() - timedelta(minutes=601) trade.orders.append( Order( ft_order_side='stoploss', @@ -2965,8 +2965,8 @@ def test_manage_open_orders_exit_usercustom( ) freqtrade = FreqtradeBot(default_conf_usdt) - open_trade_usdt.open_date = arrow.utcnow().shift(hours=-5).datetime - open_trade_usdt.close_date = arrow.utcnow().shift(minutes=-601).datetime + open_trade_usdt.open_date = dt_now() - timedelta(hours=5) + open_trade_usdt.close_date = dt_now() - timedelta(minutes=601) open_trade_usdt.close_profit_abs = 0.001 Trade.session.add(open_trade_usdt) @@ -3047,8 +3047,8 @@ def test_manage_open_orders_exit( ) freqtrade = FreqtradeBot(default_conf_usdt) - open_trade_usdt.open_date = arrow.utcnow().shift(hours=-5).datetime - open_trade_usdt.close_date = arrow.utcnow().shift(minutes=-601).datetime + open_trade_usdt.open_date = dt_now() - timedelta(hours=5) + open_trade_usdt.close_date = dt_now() - timedelta(minutes=601) open_trade_usdt.close_profit_abs = 0.001 open_trade_usdt.is_short = is_short @@ -3088,8 +3088,8 @@ def test_check_handle_cancelled_exit( ) freqtrade = FreqtradeBot(default_conf_usdt) - open_trade_usdt.open_date = arrow.utcnow().shift(hours=-5).datetime - open_trade_usdt.close_date = arrow.utcnow().shift(minutes=-601).datetime + open_trade_usdt.open_date = dt_now() - timedelta(hours=5) + open_trade_usdt.close_date = dt_now() - timedelta(minutes=601) open_trade_usdt.is_short = is_short Trade.session.add(open_trade_usdt) @@ -3417,7 +3417,7 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: exchange='binance', open_rate=0.245441, open_order_id="sell_123456", - open_date=arrow.utcnow().shift(days=-2).datetime, + open_date=dt_now() - timedelta(days=2), fee_open=fee.return_value, fee_close=fee.return_value, close_rate=0.555, From 261822147c80bba1425301b03b7d87678cc01807 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 May 2023 18:31:09 +0200 Subject: [PATCH 26/56] Fix remaining arrow testcases --- freqtrade/strategy/interface.py | 2 +- tests/commands/test_commands.py | 9 ++++----- tests/conftest.py | 29 ++++++++++++++--------------- tests/data/test_btanalysis.py | 4 ++-- tests/data/test_history.py | 17 ++++++++--------- tests/edge/test_edge.py | 5 ++--- tests/exchange/test_exchange.py | 25 ++++++++++++------------- tests/optimize/test_backtesting.py | 5 ++--- tests/optimize/test_hyperopt.py | 11 +++++------ tests/rpc/test_rpc_telegram.py | 13 ++++++------- tests/strategy/test_interface.py | 1 - tests/test_timerange.py | 1 - 12 files changed, 56 insertions(+), 66 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 80f74bfb6..382f38c9a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -954,7 +954,7 @@ class IStrategy(ABC, HyperStrategyMixin): latest_date = dataframe['date'].max() latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1] - # Explicitly convert to arrow object to ensure the below comparison does not fail + # Explicitly convert to datetime object to ensure the below comparison does not fail latest_date = latest_date.to_pydatetime() # Check if dataframe is out of date diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 318590b32..fe847e94b 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1,12 +1,11 @@ import json import re -from datetime import datetime +from datetime import datetime, timedelta from io import BytesIO from pathlib import Path from unittest.mock import MagicMock, PropertyMock from zipfile import ZipFile -import arrow import pytest from freqtrade.commands import (start_backtesting_show, start_convert_data, start_convert_trades, @@ -25,6 +24,7 @@ from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.persistence.models import init_db from freqtrade.persistence.pairlock_middleware import PairLocks +from freqtrade.util import dt_floor_day, dt_now, dt_utc from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) from tests.conftest_trades import MOCK_TRADE_COUNT @@ -689,7 +689,7 @@ def test_download_data_timerange(mocker, markets): start_download_data(pargs) assert dl_mock.call_count == 1 # 20days ago - days_ago = arrow.get(arrow.now().shift(days=-20).date()).int_timestamp + days_ago = dt_floor_day(dt_now() - timedelta(days=20)).timestamp() assert dl_mock.call_args_list[0][1]['timerange'].startts == days_ago dl_mock.reset_mock() @@ -704,8 +704,7 @@ def test_download_data_timerange(mocker, markets): start_download_data(pargs) assert dl_mock.call_count == 1 - assert dl_mock.call_args_list[0][1]['timerange'].startts == arrow.Arrow( - 2020, 1, 1).int_timestamp + assert dl_mock.call_args_list[0][1]['timerange'].startts == int(dt_utc(2020, 1, 1).timestamp()) def test_download_data_no_markets(mocker, caplog): diff --git a/tests/conftest.py b/tests/conftest.py index 55e6fa607..4d875b09d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Optional from unittest.mock import MagicMock, Mock, PropertyMock -import arrow import numpy as np import pandas as pd import pytest @@ -1693,8 +1692,8 @@ def limit_buy_order_old(): 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), 'price': 0.00001099, 'amount': 90.99181073, 'filled': 0.0, @@ -1710,8 +1709,8 @@ def limit_sell_order_old(): 'type': 'limit', 'side': 'sell', 'symbol': 'ETH/BTC', - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'price': 0.00001099, 'amount': 90.99181073, 'filled': 0.0, @@ -1727,8 +1726,8 @@ def limit_buy_order_old_partial(): 'type': 'limit', 'side': 'buy', 'symbol': 'ETH/BTC', - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'price': 0.00001099, 'amount': 90.99181073, 'filled': 23.0, @@ -1758,8 +1757,8 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': 'AZNPFF-4AC4N-7MKTAT', 'clientOrderId': None, - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': dt_now() - timedelta(minutes=601).isoformat(), 'lastTradeTimestamp': None, 'status': 'canceled', 'symbol': 'LTC/USDT', @@ -1779,8 +1778,8 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': '1234512345', 'clientOrderId': 'alb1234123', - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': dt_now() - timedelta(minutes=601).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', 'type': 'limit', @@ -1800,8 +1799,8 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': '1234512345', 'clientOrderId': 'alb1234123', - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', 'type': 'limit', @@ -2488,8 +2487,8 @@ def buy_order_fee(): 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'price': 0.245441, 'amount': 8.0, 'cost': 1.963528, diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 6550fb314..5e377f851 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -3,7 +3,6 @@ from pathlib import Path from unittest.mock import MagicMock import pytest -from arrow import Arrow from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange @@ -19,6 +18,7 @@ from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_ calculate_underwater, combine_dataframes_with_mean, create_cum_profit) from freqtrade.exceptions import OperationalException +from freqtrade.util import dt_utc from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades from tests.conftest_trades import MOCK_TRADE_COUNT @@ -421,7 +421,7 @@ def test_calculate_max_drawdown2(): -0.025782, 0.010400, 0.012374, 0.012467, 0.114741, 0.010303, 0.010088, -0.033961, 0.010680, 0.010886, -0.029274, 0.011178, 0.010693, 0.010711] - dates = [Arrow(2020, 1, 1).shift(days=i) for i in range(len(values))] + dates = [dt_utc(2020, 1, 1) + timedelta(days=i) for i in range(len(values))] df = DataFrame(zip(values, dates), columns=['profit', 'open_date']) # sort by profit and reset index df = df.sort_values('profit').reset_index(drop=True) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 24ad8bcc9..e397c97c1 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -6,7 +6,6 @@ from pathlib import Path from shutil import copyfile from unittest.mock import MagicMock, PropertyMock -import arrow import pytest from pandas import DataFrame from pandas.testing import assert_frame_equal @@ -26,6 +25,7 @@ from freqtrade.enums import CandleType from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.resolvers import StrategyResolver +from freqtrade.util import dt_utc from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, get_patched_exchange, log_has, log_has_re, patch_exchange) @@ -198,7 +198,6 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: fill_missing=False, drop_incomplete=False) # now = last cached item + 1 hour now_ts = test_data[-1][0] / 1000 + 60 * 60 - mocker.patch('arrow.utcnow', return_value=arrow.get(now_ts)) # timeframe starts earlier than the cached data # should fully update data @@ -353,10 +352,10 @@ def test_download_backtesting_data_exception(mocker, caplog, default_conf, tmpdi def test_load_partial_missing(testdatadir, caplog) -> None: # Make sure we start fresh - test missing data at start - start = arrow.get('2018-01-01T00:00:00') - end = arrow.get('2018-01-11T00:00:00') + start = dt_utc(2018, 1, 1) + end = dt_utc(2018, 1, 11) data = load_data(testdatadir, '5m', ['UNITTEST/BTC'], startup_candles=20, - timerange=TimeRange('date', 'date', start.int_timestamp, end.int_timestamp)) + timerange=TimeRange('date', 'date', start.timestamp(), end.timestamp())) assert log_has( 'Using indicator startup period: 20 ...', caplog ) @@ -369,16 +368,16 @@ def test_load_partial_missing(testdatadir, caplog) -> None: caplog) # Make sure we start fresh - test missing data at end caplog.clear() - start = arrow.get('2018-01-10T00:00:00') - end = arrow.get('2018-02-20T00:00:00') + start = dt_utc(2018, 1, 10) + end = dt_utc(2018, 2, 20) data = load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], - timerange=TimeRange('date', 'date', start.int_timestamp, end.int_timestamp)) + timerange=TimeRange('date', 'date', start.timestamp(), end.timestamp())) # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(data['UNITTEST/BTC']) # Shift endtime with +5 - end_real = arrow.get(data['UNITTEST/BTC'].iloc[-1, 0]) + end_real = data['UNITTEST/BTC'].iloc[-1, 0].to_pydatetime() assert log_has(f'UNITTEST/BTC, spot, 5m, ' f'data ends at {end_real.strftime(DATETIME_PRINT_FORMAT)}', caplog) diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index a60a785a1..2d5a308db 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -5,7 +5,6 @@ import logging import math from unittest.mock import MagicMock -import arrow import numpy as np import pytest from pandas import DataFrame @@ -14,7 +13,7 @@ from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.enums import ExitType from freqtrade.exceptions import OperationalException -from freqtrade.util.datetime_helpers import dt_ts +from freqtrade.util.datetime_helpers import dt_ts, dt_utc from tests.conftest import EXMS, get_patched_freqtradebot, log_has from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset) @@ -28,7 +27,7 @@ from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, # 5) Stoploss and sell are hit. should sell on stoploss #################################################################### -tests_start_time = arrow.get(2018, 10, 3) +tests_start_time = dt_utc(2018, 10, 3) timeframe_in_minute = 60 # End helper functions diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 7d4fbfb1a..fa2f65a14 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta, timezone from random import randint from unittest.mock import MagicMock, Mock, PropertyMock, patch -import arrow import ccxt import pytest from ccxt import DECIMAL_PLACES, ROUND, ROUND_UP, TICK_SIZE, TRUNCATE @@ -2079,7 +2078,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) ohlcv = [ [ - arrow.utcnow().int_timestamp * 1000, # unix timestamp ms + dt_ts(), # unix timestamp ms 1, # open 2, # high 3, # low @@ -2099,7 +2098,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ ret = exchange.get_historic_ohlcv( pair, "5m", - int((arrow.utcnow().int_timestamp - since) * 1000), + dt_ts(dt_now() - timedelta(seconds=since)), candle_type=candle_type ) @@ -2117,7 +2116,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ ret = exchange.get_historic_ohlcv( pair, "5m", - int((arrow.utcnow().int_timestamp - since) * 1000), + dt_ts(dt_now() - timedelta(seconds=since)), candle_type=candle_type ) assert log_has_re(r"Async code raised an exception: .*", caplog) @@ -2169,7 +2168,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None: ohlcv = [ [ - (arrow.utcnow().shift(minutes=-5).int_timestamp) * 1000, # unix timestamp ms + dt_ts(dt_now() - timedelta(minutes=5)), # unix timestamp ms 1, # open 2, # high 3, # low @@ -2177,7 +2176,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None 5, # volume (in quote currency) ], [ - arrow.utcnow().int_timestamp * 1000, # unix timestamp ms + dt_ts(), # unix timestamp ms 3, # open 1, # high 4, # low @@ -2367,7 +2366,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_name): ohlcv = [ [ - arrow.utcnow().int_timestamp * 1000, # unix timestamp ms + dt_ts(), # unix timestamp ms 1, # open 2, # high 3, # low @@ -2404,7 +2403,7 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError("Unknown error")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) await exchange._async_get_candle_history(pair, "5m", CandleType.SPOT, - (arrow.utcnow().int_timestamp - 2000) * 1000) + dt_ts(dt_now() - timedelta(seconds=2000))) exchange.close() @@ -2413,7 +2412,7 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) await exchange._async_get_candle_history(pair, "5m", CandleType.SPOT, - (arrow.utcnow().int_timestamp - 2000) * 1000) + dt_ts(dt_now() - timedelta(seconds=2000))) exchange.close() @@ -2436,7 +2435,7 @@ async def test__async_kucoin_get_candle_history(default_conf, mocker, caplog): with pytest.raises(DDosProtection, match=r'429 Too Many Requests'): await exchange._async_get_candle_history( "ETH/BTC", "5m", CandleType.SPOT, - since_ms=(arrow.utcnow().int_timestamp - 2000) * 1000, count=3) + since_ms=dt_ts(dt_now() - timedelta(seconds=2000)), count=3) assert num_log_has_re(msg, caplog) == 3 caplog.clear() @@ -2453,7 +2452,7 @@ async def test__async_kucoin_get_candle_history(default_conf, mocker, caplog): with pytest.raises(DDosProtection, match=r'429 Too Many Requests'): await exchange._async_get_candle_history( "ETH/BTC", "5m", CandleType.SPOT, - (arrow.utcnow().int_timestamp - 2000) * 1000, count=3) + dt_ts(dt_now() - timedelta(seconds=2000)), count=3) # Expect the "returned exception" message 12 times (4 retries * 3 (loop)) assert num_log_has_re(msg, caplog) == 12 assert num_log_has_re(msg2, caplog) == 9 @@ -2911,14 +2910,14 @@ async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name, with pytest.raises(OperationalException, match=r'Could not fetch trade data*'): api_mock.fetch_trades = MagicMock(side_effect=ccxt.BaseError("Unknown error")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - await exchange._async_fetch_trades(pair, since=(arrow.utcnow().int_timestamp - 2000) * 1000) + await exchange._async_fetch_trades(pair, since=dt_ts(dt_now() - timedelta(seconds=2000))) exchange.close() with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching ' r'historical trade data\..*'): api_mock.fetch_trades = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - await exchange._async_fetch_trades(pair, since=(arrow.utcnow().int_timestamp - 2000) * 1000) + await exchange._async_fetch_trades(pair, since=dt_ts(dt_now() - timedelta(seconds=2000))) exchange.close() diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 0af75111a..bef942b43 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -9,7 +9,6 @@ from unittest.mock import MagicMock, PropertyMock import numpy as np import pandas as pd import pytest -from arrow import Arrow from freqtrade import constants from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting @@ -347,7 +346,7 @@ def test_backtest_abort(default_conf, mocker, testdatadir) -> None: def test_backtesting_start(default_conf, mocker, caplog) -> None: def get_timerange(input1): - return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) + return dt_utc(2017, 11, 14, 21, 17), dt_utc(2017, 11, 14, 22, 59) mocker.patch('freqtrade.data.history.get_timerange', get_timerange) patch_exchange(mocker) @@ -386,7 +385,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: def get_timerange(input1): - return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) + return dt_utc(2017, 11, 14, 21, 17), dt_utc(2017, 11, 14, 22, 59) mocker.patch('freqtrade.data.history.history_utils.load_pair_history', MagicMock(return_value=pd.DataFrame())) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 63691b08d..ed5eeafd6 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -6,7 +6,6 @@ from unittest.mock import ANY, MagicMock, PropertyMock import pandas as pd import pytest -from arrow import Arrow from filelock import Timeout from skopt.space import Integer @@ -380,8 +379,8 @@ def test_hyperopt_format_results(hyperopt): 'backtest_end_time': 1619718665, } results_metrics = generate_strategy_stats(['XRP/BTC'], '', bt_result, - Arrow(2017, 11, 14, 19, 32, 00), - Arrow(2017, 12, 14, 19, 32, 00), market_change=0) + dt_utc(2017, 11, 14, 19, 32, 00), + dt_utc(2017, 12, 14, 19, 32, 00), market_change=0) results_explanation = HyperoptTools.format_results_explanation_string(results_metrics, 'BTC') total_profit = results_metrics['profit_total_abs'] @@ -454,7 +453,7 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: mocker.patch('freqtrade.optimize.hyperopt.Backtesting.backtest', return_value=backtest_result) mocker.patch('freqtrade.optimize.hyperopt.get_timerange', - return_value=(Arrow(2017, 12, 10), Arrow(2017, 12, 13))) + return_value=(dt_utc(2017, 12, 10), dt_utc(2017, 12, 13))) patch_exchange(mocker) mocker.patch.object(Path, 'open') mocker.patch('freqtrade.configuration.config_validation.validate_config_schema') @@ -514,8 +513,8 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: } hyperopt = Hyperopt(hyperopt_conf) - hyperopt.min_date = Arrow(2017, 12, 10) - hyperopt.max_date = Arrow(2017, 12, 13) + hyperopt.min_date = dt_utc(2017, 12, 10) + hyperopt.max_date = dt_utc(2017, 12, 13) hyperopt.init_spaces() generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values())) assert generate_optimizer_value == response_expected diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 96d56d13f..51879f5ad 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -12,7 +12,6 @@ from random import choice, randint from string import ascii_uppercase from unittest.mock import ANY, AsyncMock, MagicMock -import arrow import pytest import time_machine from pandas import DataFrame @@ -260,7 +259,7 @@ async def test_telegram_status(default_conf, update, mocker) -> None: 'pair': 'ETH/BTC', 'base_currency': 'ETH', 'quote_currency': 'BTC', - 'open_date': arrow.utcnow(), + 'open_date': dt_now(), 'close_date': None, 'open_rate': 1.099e-05, 'close_rate': None, @@ -2073,7 +2072,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, 'open_date': dt_now() - timedelta(hours=1), - 'close_date': arrow.utcnow(), + 'close_date': dt_now(), }) assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' @@ -2109,7 +2108,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), - 'close_date': arrow.utcnow(), + 'close_date': dt_now(), 'stake_amount': 0.01, 'sub_trade': True, }) @@ -2146,7 +2145,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), - 'close_date': arrow.utcnow(), + 'close_date': dt_now(), }) assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' @@ -2228,7 +2227,7 @@ def test_send_msg_sell_fill_notification(default_conf, mocker, direction, 'enter_tag': enter_signal, 'exit_reason': ExitType.STOP_LOSS.value, 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), - 'close_date': arrow.utcnow(), + 'close_date': dt_now(), }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' @@ -2365,7 +2364,7 @@ def test_send_msg_sell_notification_no_fiat( 'enter_tag': enter_signal, 'exit_reason': ExitType.STOP_LOSS.value, 'open_date': dt_now() - timedelta(hours=2, minutes=35, seconds=3), - 'close_date': arrow.utcnow(), + 'close_date': dt_now(), }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 8be0986cd..8a609cf30 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -4,7 +4,6 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import MagicMock -import arrow import pytest from pandas import DataFrame diff --git a/tests/test_timerange.py b/tests/test_timerange.py index 8247b60be..d1c61704f 100644 --- a/tests/test_timerange.py +++ b/tests/test_timerange.py @@ -1,7 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103 from datetime import datetime, timezone -import arrow import pytest from freqtrade.configuration import TimeRange From adcf7513408055e843c3f2a74bf3cd1f31810b27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 15 May 2023 06:25:06 +0200 Subject: [PATCH 27/56] Bump min-requirement of arrow --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b59e98ae8..f8b8b515c 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setup( 'ccxt>=3.0.0', 'SQLAlchemy>=2.0.6', 'python-telegram-bot>=20.1', - 'arrow>=0.17.0', + 'arrow>=1.0.0', 'cachetools', 'requests', 'urllib3', From b40c45ee4246885ab3b2319bd9566ee417d4a0e9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 15 May 2023 19:39:49 +0200 Subject: [PATCH 28/56] Timerange -> datetime --- freqtrade/configuration/timerange.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index 0c2f0d1b8..cff35db7e 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -6,8 +6,6 @@ import re from datetime import datetime, timezone from typing import Optional -import arrow - from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.exceptions import OperationalException @@ -139,7 +137,8 @@ class TimeRange: if stype[0]: starts = rvals[index] if stype[0] == 'date' and len(starts) == 8: - start = arrow.get(starts, 'YYYYMMDD').int_timestamp + start = int(datetime.strptime(starts, '%Y%m%d').replace( + tzinfo=timezone.utc).timestamp()) elif len(starts) == 13: start = int(starts) // 1000 else: @@ -148,7 +147,8 @@ class TimeRange: if stype[1]: stops = rvals[index] if stype[1] == 'date' and len(stops) == 8: - stop = arrow.get(stops, 'YYYYMMDD').int_timestamp + stop = int(datetime.strptime(stops, '%Y%m%d').replace( + tzinfo=timezone.utc).timestamp()) elif len(stops) == 13: stop = int(stops) // 1000 else: From f657d06e91e2f5e53eedc3c972de4b0439fb6d64 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 15 May 2023 19:55:54 +0200 Subject: [PATCH 29/56] Move shorten_date to datetime helpers --- freqtrade/misc.py | 13 ------------- freqtrade/rpc/rpc.py | 4 ++-- freqtrade/util/__init__.py | 4 +++- freqtrade/util/datetime_helpers.py | 13 +++++++++++++ tests/test_misc.py | 8 +------- tests/utils/test_datetime_helpers.py | 8 +++++++- 6 files changed, 26 insertions(+), 24 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 0cd5c6ffd..d2b0bfd08 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -3,7 +3,6 @@ Various tool function for Freqtrade and scripts """ import gzip import logging -import re from datetime import datetime from pathlib import Path from typing import Any, Dict, Iterator, List, Mapping, Optional, TextIO, Union @@ -48,18 +47,6 @@ def round_coin_value( return val -def shorten_date(_date: str) -> str: - """ - Trim the date so it fits on small screens - """ - new_date = re.sub('seconds?', 'sec', _date) - new_date = re.sub('minutes?', 'min', new_date) - new_date = re.sub('hours?', 'h', new_date) - new_date = re.sub('days?', 'd', new_date) - new_date = re.sub('^an?', '1', new_date) - return new_date - - def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = True) -> None: """ Dump JSON data into a file diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 245575eb7..28b210ad9 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -26,13 +26,13 @@ from freqtrade.exceptions import ExchangeError, PricingError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.exchange.types import Tickers from freqtrade.loggers import bufferHandler -from freqtrade.misc import decimals_per_coin, shorten_date +from freqtrade.misc import decimals_per_coin from freqtrade.persistence import KeyStoreKeys, KeyValueStore, Order, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.rpc.rpc_types import RPCSendMsg -from freqtrade.util.datetime_helpers import dt_now +from freqtrade.util import dt_now, shorten_date from freqtrade.wallets import PositionWallet, Wallet diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index aa0ad05f2..68cb724e9 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,4 +1,5 @@ -from freqtrade.util.datetime_helpers import dt_floor_day, dt_from_ts, dt_now, dt_ts, dt_utc +from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_now, dt_ts, dt_utc, + shorten_date) from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache @@ -9,6 +10,7 @@ __all__ = [ 'dt_now', 'dt_ts', 'dt_utc', + 'shorten_date', 'FtPrecise', 'PeriodicCache', ] diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index c44761fd4..ffe8effaa 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -1,3 +1,4 @@ +import re from datetime import datetime, timezone from typing import Optional @@ -37,3 +38,15 @@ def dt_from_ts(timestamp: float) -> datetime: # Timezone in ms - convert to seconds timestamp /= 1000 return datetime.fromtimestamp(timestamp, tz=timezone.utc) + + +def shorten_date(_date: str) -> str: + """ + Trim the date so it fits on small screens + """ + new_date = re.sub('seconds?', 'sec', _date) + new_date = re.sub('minutes?', 'min', new_date) + new_date = re.sub('hours?', 'h', new_date) + new_date = re.sub('days?', 'd', new_date) + new_date = re.sub('^an?', '1', new_date) + return new_date diff --git a/tests/test_misc.py b/tests/test_misc.py index 6b4343ab2..03a236d73 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -12,7 +12,7 @@ from freqtrade.misc import (dataframe_to_json, decimals_per_coin, deep_merge_dic file_load_json, format_ms_time, json_to_dataframe, pair_to_filename, parse_db_uri_for_logging, plural, render_template, render_template_with_fallback, round_coin_value, safe_value_fallback, - safe_value_fallback2, shorten_date) + safe_value_fallback2) def test_decimals_per_coin(): @@ -39,12 +39,6 @@ def test_round_coin_value(): assert round_coin_value(222.2, 'USDT', False, True) == '222.200' -def test_shorten_date() -> None: - str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago' - str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago' - assert shorten_date(str_data) == str_shorten_data - - def test_file_dump_json(mocker) -> None: file_open = mocker.patch('freqtrade.misc.Path.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index f1c445caf..7a596fcbe 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta, timezone import pytest import time_machine -from freqtrade.util import dt_floor_day, dt_from_ts, dt_now, dt_ts, dt_utc +from freqtrade.util import dt_floor_day, dt_from_ts, dt_now, dt_ts, dt_utc, shorten_date def test_dt_now(): @@ -45,3 +45,9 @@ def test_dt_floor_day(): now = datetime(2023, 9, 1, 5, 2, 3, 455555, tzinfo=timezone.utc) assert dt_floor_day(now) == datetime(2023, 9, 1, tzinfo=timezone.utc) + + +def test_shorten_date() -> None: + str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago' + str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago' + assert shorten_date(str_data) == str_shorten_data From 5d0cff2f763e18d2c32903d5db8c24c2c20ef26a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 May 2023 07:07:22 +0200 Subject: [PATCH 30/56] Add dt_humanize helper --- freqtrade/exchange/exchange.py | 4 ++-- freqtrade/util/__init__.py | 5 +++-- freqtrade/util/datetime_helpers.py | 11 +++++++++++ tests/utils/test_datetime_helpers.py | 7 +++++++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 1c287438d..e41034f44 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -43,7 +43,7 @@ from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_ safe_value_fallback2) from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.util import dt_from_ts, dt_now -from freqtrade.util.datetime_helpers import dt_ts +from freqtrade.util.datetime_helpers import dt_humanize, dt_ts logger = logging.getLogger(__name__) @@ -1932,7 +1932,7 @@ class Exchange: logger.debug( "one_call: %s msecs (%s)", one_call, - arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) + dt_humanize(dt_now() - timedelta(milliseconds=one_call), only_distance=True) ) input_coroutines = [self._async_get_candle_history( pair, timeframe, candle_type, since) for since in diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 68cb724e9..bed65a54b 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,5 +1,5 @@ -from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_now, dt_ts, dt_utc, - shorten_date) +from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, + dt_utc, shorten_date) from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache @@ -10,6 +10,7 @@ __all__ = [ 'dt_now', 'dt_ts', 'dt_utc', + 'dt_humanize', 'shorten_date', 'FtPrecise', 'PeriodicCache', diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index ffe8effaa..39d134e11 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -2,6 +2,8 @@ import re from datetime import datetime, timezone from typing import Optional +import arrow + def dt_now() -> datetime: """Return the current datetime in UTC.""" @@ -50,3 +52,12 @@ def shorten_date(_date: str) -> str: new_date = re.sub('days?', 'd', new_date) new_date = re.sub('^an?', '1', new_date) return new_date + + +def dt_humanize(dt: datetime, **kwargs) -> str: + """ + Return a humanized string for the given datetime. + :param dt: datetime to humanize + :param kwargs: kwargs to pass to arrow's humanize() + """ + return arrow.get(dt).humanize(**kwargs) diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index 7a596fcbe..0db5f3798 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -4,6 +4,7 @@ import pytest import time_machine from freqtrade.util import dt_floor_day, dt_from_ts, dt_now, dt_ts, dt_utc, shorten_date +from freqtrade.util.datetime_helpers import dt_humanize def test_dt_now(): @@ -51,3 +52,9 @@ def test_shorten_date() -> None: str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago' str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago' assert shorten_date(str_data) == str_shorten_data + + +def test_dt_humanize() -> None: + assert dt_humanize(dt_now()) == 'just now' + assert dt_humanize(dt_now(), only_distance=True) == 'instantly' + assert dt_humanize(dt_now() - timedelta(hours=16), only_distance=True) == '16 hours' From ebfc9a6039d9e48467c06a7818464fb344ffedc1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 May 2023 19:26:31 +0200 Subject: [PATCH 31/56] Remove some humanize occurances --- freqtrade/rpc/rpc.py | 9 ++++----- freqtrade/rpc/telegram.py | 18 +++++------------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 28b210ad9..dedb35503 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -7,7 +7,6 @@ from datetime import date, datetime, timedelta, timezone from math import isnan from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple, Union -import arrow import psutil from dateutil.relativedelta import relativedelta from dateutil.tz import tzlocal @@ -32,7 +31,7 @@ from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.rpc.rpc_types import RPCSendMsg -from freqtrade.util import dt_now, shorten_date +from freqtrade.util import dt_humanize, dt_now, shorten_date from freqtrade.wallets import PositionWallet, Wallet @@ -293,7 +292,7 @@ class RPC: and open_order.ft_order_side == trade.entry_side) else '') + ('**' if (open_order and open_order.ft_order_side == trade.exit_side is not None) else ''), - shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), + shorten_date(dt_humanize(trade.open_date, only_distance=True)), profit_str ] if self._config.get('position_adjustment_enable', False): @@ -565,10 +564,10 @@ class RPC: 'trade_count': len(trades), 'closed_trade_count': len([t for t in trades if not t.is_open]), 'first_trade_date': first_date.strftime(DATETIME_PRINT_FORMAT) if first_date else '', - 'first_trade_humanized': arrow.get(first_date).humanize() if first_date else '', + 'first_trade_humanized': dt_humanize(first_date) if first_date else '', 'first_trade_timestamp': int(first_date.timestamp() * 1000) if first_date else 0, 'latest_trade_date': last_date.strftime(DATETIME_PRINT_FORMAT) if last_date else '', - 'latest_trade_humanized': arrow.get(last_date).humanize() if last_date else '', + 'latest_trade_humanized': dt_humanize(last_date) if last_date else '', 'latest_trade_timestamp': int(last_date.timestamp() * 1000) if last_date else 0, 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], 'best_pair': best_pair[0] if best_pair else '', diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 9ecc2b677..07189e29b 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -34,6 +34,7 @@ from freqtrade.misc import chunks, plural, round_coin_value from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException, RPCHandler from freqtrade.rpc.rpc_types import RPCSendMsg +from freqtrade.util import dt_humanize MAX_MESSAGE_LENGTH = MessageLimit.MAX_TEXT_LENGTH @@ -528,7 +529,6 @@ class Telegram(RPCHandler): order_nr += 1 wording = 'Entry' if order['ft_is_entry'] else 'Exit' - cur_entry_datetime = arrow.get(order["order_filled_date"]) cur_entry_amount = order["filled"] or order["amount"] cur_entry_average = order["safe_price"] lines.append(" ") @@ -559,22 +559,14 @@ class Telegram(RPCHandler): lines.append(f"*{wording} #{order_nr}:* at {minus_on_entry:.2%} avg Profit") if is_open: - lines.append("({})".format(cur_entry_datetime - .humanize(granularity=["day", "hour", "minute"]))) + lines.append("({})".format(dt_humanize(order["order_filled_date"], + granularity=["day", "hour", "minute"]))) lines.append(f"*Amount:* {cur_entry_amount} " f"({round_coin_value(order['cost'], quote_currency)})") lines.append(f"*Average {wording} Price:* {cur_entry_average} " f"({price_to_1st_entry:.2%} from 1st entry Rate)") lines.append(f"*Order filled:* {order['order_filled_date']}") - # TODO: is this really useful? - # dur_entry = cur_entry_datetime - arrow.get( - # filled_orders[x - 1]["order_filled_date"]) - # days = dur_entry.days - # hours, remainder = divmod(dur_entry.seconds, 3600) - # minutes, seconds = divmod(remainder, 60) - # lines.append( - # f"({days}d {hours}h {minutes}m {seconds}s from previous {wording.lower()})") lines_detail.append("\n".join(lines)) return lines_detail @@ -610,7 +602,7 @@ class Telegram(RPCHandler): position_adjust = self._config.get('position_adjustment_enable', False) max_entries = self._config.get('max_entry_position_adjustment', -1) for r in results: - r['open_date_hum'] = arrow.get(r['open_date']).humanize() + r['open_date_hum'] = dt_humanize(r['open_date']) r['num_entries'] = len([o for o in r['orders'] if o['ft_is_entry']]) r['num_exits'] = len([o for o in r['orders'] if not o['ft_is_entry'] and not o['ft_order_side'] == 'stoploss']) @@ -1219,7 +1211,7 @@ class Telegram(RPCHandler): nrecent ) trades_tab = tabulate( - [[arrow.get(trade['close_date']).humanize(), + [[dt_humanize(trade['close_date']), trade['pair'] + " (#" + str(trade['trade_id']) + ")", f"{(trade['close_profit']):.2%} ({trade['close_profit_abs']})"] for trade in trades['trades']], From 9d0f488de7be653152d760b946e60f548e850631 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 May 2023 19:48:11 +0200 Subject: [PATCH 32/56] Some more edits due to arrow --- freqtrade/exchange/exchange.py | 1 - freqtrade/persistence/trade_model.py | 4 ++-- freqtrade/rpc/telegram.py | 1 - tests/conftest.py | 4 ++-- tests/edge/test_edge.py | 5 +++-- tests/utils/test_datetime_helpers.py | 1 - 6 files changed, 7 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e41034f44..e26ae7c86 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -11,7 +11,6 @@ from math import floor from threading import Lock from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union -import arrow import ccxt import ccxt.async_support as ccxt_async from cachetools import TTLCache diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index cc72e2bf0..5d8aada6b 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -19,7 +19,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi price_to_precision) from freqtrade.leverage import interest from freqtrade.persistence.base import ModelBase, SessionType -from freqtrade.util import FtPrecise +from freqtrade.util import FtPrecise, dt_now logger = logging.getLogger(__name__) @@ -68,7 +68,7 @@ class Order(ModelBase): remaining: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) stop_price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) - order_date: Mapped[datetime] = mapped_column(nullable=True, default=datetime.utcnow) + order_date: Mapped[datetime] = mapped_column(nullable=True, default=dt_now) order_filled_date: Mapped[Optional[datetime]] = mapped_column(nullable=True) order_update_date: Mapped[Optional[datetime]] = mapped_column(nullable=True) funding_fee: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 07189e29b..d082299cb 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -17,7 +17,6 @@ from math import isnan from threading import Thread from typing import Any, Callable, Coroutine, Dict, List, Optional, Union -import arrow from tabulate import tabulate from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, ReplyKeyboardMarkup, Update) diff --git a/tests/conftest.py b/tests/conftest.py index 4d875b09d..66f331cae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1758,7 +1758,7 @@ def limit_buy_order_canceled_empty(request): 'id': 'AZNPFF-4AC4N-7MKTAT', 'clientOrderId': None, 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), - 'datetime': dt_now() - timedelta(minutes=601).isoformat(), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'lastTradeTimestamp': None, 'status': 'canceled', 'symbol': 'LTC/USDT', @@ -1779,7 +1779,7 @@ def limit_buy_order_canceled_empty(request): 'id': '1234512345', 'clientOrderId': 'alb1234123', 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), - 'datetime': dt_now() - timedelta(minutes=601).isoformat(), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', 'type': 'limit', diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index 2d5a308db..4829dd035 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -3,6 +3,7 @@ import logging import math +from datetime import timedelta from unittest.mock import MagicMock import numpy as np @@ -232,7 +233,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', NEOBTC = [ [ - tests_start_time.shift(minutes=(x * timeframe_in_minute)).int_timestamp * 1000, + dt_ts(tests_start_time + timedelta(minutes=(x * timeframe_in_minute))), math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, @@ -244,7 +245,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', base = 0.002 LTCBTC = [ [ - tests_start_time.shift(minutes=(x * timeframe_in_minute)).int_timestamp * 1000, + dt_ts(tests_start_time + timedelta(minutes=(x * timeframe_in_minute))), math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index 0db5f3798..5aec0da54 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -28,7 +28,6 @@ def test_dt_utc(): tzinfo=timezone.utc) - @pytest.mark.parametrize('as_ms', [True, False]) def test_dt_from_ts(as_ms): multi = 1000 if as_ms else 1 From 53162272193cf9b5dc233650022c1cf9ff51e94c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:08:52 +0200 Subject: [PATCH 33/56] Extract api backtest logic from ApiServer class --- freqtrade/rpc/api_server/api_backtest.py | 86 ++++++++++---------- freqtrade/rpc/api_server/deps.py | 7 +- freqtrade/rpc/api_server/webserver.py | 15 +--- freqtrade/rpc/api_server/webserver_bgwork.py | 16 ++++ tests/rpc/test_rpc_apiserver.py | 5 +- 5 files changed, 68 insertions(+), 61 deletions(-) create mode 100644 freqtrade/rpc/api_server/webserver_bgwork.py diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index b168affc3..4055845e1 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -16,7 +16,7 @@ from freqtrade.misc import deep_merge_dicts from freqtrade.rpc.api_server.api_schemas import (BacktestHistoryEntry, BacktestRequest, BacktestResponse) from freqtrade.rpc.api_server.deps import get_config, is_webserver_mode -from freqtrade.rpc.api_server.webserver import ApiServer +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from freqtrade.rpc.rpc import RPCException @@ -30,9 +30,9 @@ router = APIRouter() async def api_start_backtest( # noqa: C901 bt_settings: BacktestRequest, background_tasks: BackgroundTasks, config=Depends(get_config), ws_mode=Depends(is_webserver_mode)): - ApiServer._bt['bt_error'] = None + ApiBG._bt['bt_error'] = None """Start backtesting if not done so already""" - if ApiServer._bgtask_running: + if ApiBG._bgtask_running: raise RPCException('Bot Background task already running') if ':' in bt_settings.strategy: @@ -63,30 +63,30 @@ async def api_start_backtest( # noqa: C901 asyncio.set_event_loop(asyncio.new_event_loop()) try: # Reload strategy - lastconfig = ApiServer._bt['last_config'] + lastconfig = ApiBG._bt['last_config'] strat = StrategyResolver.load_strategy(btconfig) validate_config_consistency(btconfig) if ( - not ApiServer._bt['bt'] + not ApiBG._bt['bt'] or lastconfig.get('timeframe') != strat.timeframe or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') or lastconfig.get('timerange') != btconfig['timerange'] ): from freqtrade.optimize.backtesting import Backtesting - ApiServer._bt['bt'] = Backtesting(btconfig) - ApiServer._bt['bt'].load_bt_data_detail() + ApiBG._bt['bt'] = Backtesting(btconfig) + ApiBG._bt['bt'].load_bt_data_detail() else: - ApiServer._bt['bt'].config = btconfig - ApiServer._bt['bt'].init_backtest() + ApiBG._bt['bt'].config = btconfig + ApiBG._bt['bt'].init_backtest() # Only reload data if timeframe changed. if ( - not ApiServer._bt['data'] - or not ApiServer._bt['timerange'] + not ApiBG._bt['data'] + or not ApiBG._bt['timerange'] or lastconfig.get('timeframe') != strat.timeframe or lastconfig.get('timerange') != btconfig['timerange'] ): - ApiServer._bt['data'], ApiServer._bt['timerange'] = ApiServer._bt[ + ApiBG._bt['data'], ApiBG._bt['timerange'] = ApiBG._bt[ 'bt'].load_bt_data() lastconfig['timerange'] = btconfig['timerange'] @@ -95,27 +95,27 @@ async def api_start_backtest( # noqa: C901 lastconfig['enable_protections'] = btconfig.get('enable_protections') lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') - ApiServer._bt['bt'].enable_protections = btconfig.get('enable_protections', False) - ApiServer._bt['bt'].strategylist = [strat] - ApiServer._bt['bt'].results = {} - ApiServer._bt['bt'].load_prior_backtest() + ApiBG._bt['bt'].enable_protections = btconfig.get('enable_protections', False) + ApiBG._bt['bt'].strategylist = [strat] + ApiBG._bt['bt'].results = {} + ApiBG._bt['bt'].load_prior_backtest() - ApiServer._bt['bt'].abort = False - if (ApiServer._bt['bt'].results and - strat.get_strategy_name() in ApiServer._bt['bt'].results['strategy']): + ApiBG._bt['bt'].abort = False + if (ApiBG._bt['bt'].results and + strat.get_strategy_name() in ApiBG._bt['bt'].results['strategy']): # When previous result hash matches - reuse that result and skip backtesting. logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}') else: - min_date, max_date = ApiServer._bt['bt'].backtest_one_strategy( - strat, ApiServer._bt['data'], ApiServer._bt['timerange']) + min_date, max_date = ApiBG._bt['bt'].backtest_one_strategy( + strat, ApiBG._bt['data'], ApiBG._bt['timerange']) - ApiServer._bt['bt'].results = generate_backtest_stats( - ApiServer._bt['data'], ApiServer._bt['bt'].all_results, + ApiBG._bt['bt'].results = generate_backtest_stats( + ApiBG._bt['data'], ApiBG._bt['bt'].all_results, min_date=min_date, max_date=max_date) if btconfig.get('export', 'none') == 'trades': store_backtest_stats( - btconfig['exportfilename'], ApiServer._bt['bt'].results, + btconfig['exportfilename'], ApiBG._bt['bt'].results, datetime.now().strftime("%Y-%m-%d_%H-%M-%S") ) @@ -123,13 +123,13 @@ async def api_start_backtest( # noqa: C901 except (Exception, OperationalException, DependencyException) as e: logger.exception(f"Backtesting caused an error: {e}") - ApiServer._bt['bt_error'] = str(e) + ApiBG._bt['bt_error'] = str(e) pass finally: - ApiServer._bgtask_running = False + ApiBG._bgtask_running = False background_tasks.add_task(run_backtest) - ApiServer._bgtask_running = True + ApiBG._bgtask_running = True return { "status": "running", @@ -147,18 +147,18 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): Returns Result after backtesting has been ran. """ from freqtrade.persistence import LocalTrade - if ApiServer._bgtask_running: + if ApiBG._bgtask_running: return { "status": "running", "running": True, - "step": (ApiServer._bt['bt'].progress.action if ApiServer._bt['bt'] + "step": (ApiBG._bt['bt'].progress.action if ApiBG._bt['bt'] else str(BacktestState.STARTUP)), - "progress": ApiServer._bt['bt'].progress.progress if ApiServer._bt['bt'] else 0, + "progress": ApiBG._bt['bt'].progress.progress if ApiBG._bt['bt'] else 0, "trade_count": len(LocalTrade.trades), "status_msg": "Backtest running", } - if not ApiServer._bt['bt']: + if not ApiBG._bt['bt']: return { "status": "not_started", "running": False, @@ -166,13 +166,13 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest not yet executed" } - if ApiServer._bt['bt_error']: + if ApiBG._bt['bt_error']: return { "status": "error", "running": False, "step": "", "progress": 0, - "status_msg": f"Backtest failed with {ApiServer._bt['bt_error']}" + "status_msg": f"Backtest failed with {ApiBG._bt['bt_error']}" } return { @@ -181,14 +181,14 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): "status_msg": "Backtest ended", "step": "finished", "progress": 1, - "backtest_result": ApiServer._bt['bt'].results, + "backtest_result": ApiBG._bt['bt'].results, } @router.delete('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest']) def api_delete_backtest(ws_mode=Depends(is_webserver_mode)): """Reset backtesting""" - if ApiServer._bgtask_running: + if ApiBG._bgtask_running: return { "status": "running", "running": True, @@ -196,12 +196,12 @@ def api_delete_backtest(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest running", } - if ApiServer._bt['bt']: - ApiServer._bt['bt'].cleanup() - del ApiServer._bt['bt'] - ApiServer._bt['bt'] = None - del ApiServer._bt['data'] - ApiServer._bt['data'] = None + if ApiBG._bt['bt']: + ApiBG._bt['bt'].cleanup() + del ApiBG._bt['bt'] + ApiBG._bt['bt'] = None + del ApiBG._bt['data'] + ApiBG._bt['data'] = None logger.info("Backtesting reset") return { "status": "reset", @@ -214,7 +214,7 @@ def api_delete_backtest(ws_mode=Depends(is_webserver_mode)): @router.get('/backtest/abort', response_model=BacktestResponse, tags=['webserver', 'backtest']) def api_backtest_abort(ws_mode=Depends(is_webserver_mode)): - if not ApiServer._bgtask_running: + if not ApiBG._bgtask_running: return { "status": "not_running", "running": False, @@ -222,7 +222,7 @@ def api_backtest_abort(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest ended", } - ApiServer._bt['bt'].abort = True + ApiBG._bt['bt'].abort = True return { "status": "stopping", "running": False, diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index bfc1e698c..b46ac88cd 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -6,6 +6,7 @@ from fastapi import Depends from freqtrade.enums import RunMode from freqtrade.persistence import Trade from freqtrade.persistence.models import _request_id_ctx_var +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from freqtrade.rpc.rpc import RPC, RPCException from .webserver import ApiServer @@ -43,11 +44,11 @@ def get_api_config() -> Dict[str, Any]: def get_exchange(config=Depends(get_config)): - if not ApiServer._exchange: + if not ApiBG._exchange: from freqtrade.resolvers import ExchangeResolver - ApiServer._exchange = ExchangeResolver.load_exchange( + ApiBG._exchange = ExchangeResolver.load_exchange( config, load_leverage_tiers=False) - return ApiServer._exchange + return ApiBG._exchange def get_message_stream(): diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 8030e303b..165849a7f 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -1,6 +1,6 @@ import logging from ipaddress import IPv4Address -from typing import Any, Dict, Optional +from typing import Any, Optional import orjson import uvicorn @@ -36,19 +36,8 @@ class ApiServer(RPCHandler): __initialized = False _rpc: RPC - # Backtesting type: Backtesting - _bt: Dict[str, Any] = { - 'bt': None, - 'data': None, - 'timerange': None, - 'last_config': {}, - 'bt_error': None, - } _has_rpc: bool = False - _bgtask_running: bool = False _config: Config = {} - # Exchange - only available in webserver mode. - _exchange = None # websocket message stuff _message_stream: Optional[MessageStream] = None @@ -85,7 +74,7 @@ class ApiServer(RPCHandler): """ Attach rpc handler """ - if not self._has_rpc: + if not ApiServer._has_rpc: ApiServer._rpc = rpc ApiServer._has_rpc = True else: diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py new file mode 100644 index 000000000..cdb1bd031 --- /dev/null +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -0,0 +1,16 @@ + +from typing import Any, Dict + + +class ApiBG(): + # Backtesting type: Backtesting + _bt: Dict[str, Any] = { + 'bt': None, + 'data': None, + 'timerange': None, + 'last_config': {}, + 'bt_error': None, + } + _bgtask_running: bool = False + # Exchange - only available in webserver mode. + _exchange = None diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 1e76ce557..aad4bedb6 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -26,6 +26,7 @@ from freqtrade.rpc import RPC from freqtrade.rpc.api_server import ApiServer from freqtrade.rpc.api_server.api_auth import create_token, get_user_from_token from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, create_mock_trades, get_mock_coro, get_patched_freqtradebot, log_has, log_has_re, patch_get_signal) @@ -1733,7 +1734,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): assert result['status_msg'] == 'Backtest ended' # Simulate running backtest - ApiServer._bgtask_running = True + ApiBG._bgtask_running = True rc = client_get(client, f"{BASE_URI}/backtest/abort") assert_response(rc) result = rc.json() @@ -1762,7 +1763,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): result = rc.json() assert 'Bot Background task already running' in result['error'] - ApiServer._bgtask_running = False + ApiBG._bgtask_running = False # Rerun backtest (should get previous result) rc = client_post(client, f"{BASE_URI}/backtest", data=data) From 96d74063fcadd444db473aa187246633b25a25d7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:12:02 +0200 Subject: [PATCH 34/56] Don't have public attributes marked as private --- freqtrade/rpc/api_server/api_backtest.py | 84 ++++++++++---------- freqtrade/rpc/api_server/deps.py | 6 +- freqtrade/rpc/api_server/webserver_bgwork.py | 6 +- tests/rpc/test_rpc_apiserver.py | 4 +- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 4055845e1..a06d65dcc 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -30,9 +30,9 @@ router = APIRouter() async def api_start_backtest( # noqa: C901 bt_settings: BacktestRequest, background_tasks: BackgroundTasks, config=Depends(get_config), ws_mode=Depends(is_webserver_mode)): - ApiBG._bt['bt_error'] = None + ApiBG.bt['bt_error'] = None """Start backtesting if not done so already""" - if ApiBG._bgtask_running: + if ApiBG.bgtask_running: raise RPCException('Bot Background task already running') if ':' in bt_settings.strategy: @@ -63,30 +63,30 @@ async def api_start_backtest( # noqa: C901 asyncio.set_event_loop(asyncio.new_event_loop()) try: # Reload strategy - lastconfig = ApiBG._bt['last_config'] + lastconfig = ApiBG.bt['last_config'] strat = StrategyResolver.load_strategy(btconfig) validate_config_consistency(btconfig) if ( - not ApiBG._bt['bt'] + not ApiBG.bt['bt'] or lastconfig.get('timeframe') != strat.timeframe or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') or lastconfig.get('timerange') != btconfig['timerange'] ): from freqtrade.optimize.backtesting import Backtesting - ApiBG._bt['bt'] = Backtesting(btconfig) - ApiBG._bt['bt'].load_bt_data_detail() + ApiBG.bt['bt'] = Backtesting(btconfig) + ApiBG.bt['bt'].load_bt_data_detail() else: - ApiBG._bt['bt'].config = btconfig - ApiBG._bt['bt'].init_backtest() + ApiBG.bt['bt'].config = btconfig + ApiBG.bt['bt'].init_backtest() # Only reload data if timeframe changed. if ( - not ApiBG._bt['data'] - or not ApiBG._bt['timerange'] + not ApiBG.bt['data'] + or not ApiBG.bt['timerange'] or lastconfig.get('timeframe') != strat.timeframe or lastconfig.get('timerange') != btconfig['timerange'] ): - ApiBG._bt['data'], ApiBG._bt['timerange'] = ApiBG._bt[ + ApiBG.bt['data'], ApiBG.bt['timerange'] = ApiBG.bt[ 'bt'].load_bt_data() lastconfig['timerange'] = btconfig['timerange'] @@ -95,27 +95,27 @@ async def api_start_backtest( # noqa: C901 lastconfig['enable_protections'] = btconfig.get('enable_protections') lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') - ApiBG._bt['bt'].enable_protections = btconfig.get('enable_protections', False) - ApiBG._bt['bt'].strategylist = [strat] - ApiBG._bt['bt'].results = {} - ApiBG._bt['bt'].load_prior_backtest() + ApiBG.bt['bt'].enable_protections = btconfig.get('enable_protections', False) + ApiBG.bt['bt'].strategylist = [strat] + ApiBG.bt['bt'].results = {} + ApiBG.bt['bt'].load_prior_backtest() - ApiBG._bt['bt'].abort = False - if (ApiBG._bt['bt'].results and - strat.get_strategy_name() in ApiBG._bt['bt'].results['strategy']): + ApiBG.bt['bt'].abort = False + if (ApiBG.bt['bt'].results and + strat.get_strategy_name() in ApiBG.bt['bt'].results['strategy']): # When previous result hash matches - reuse that result and skip backtesting. logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}') else: - min_date, max_date = ApiBG._bt['bt'].backtest_one_strategy( - strat, ApiBG._bt['data'], ApiBG._bt['timerange']) + min_date, max_date = ApiBG.bt['bt'].backtest_one_strategy( + strat, ApiBG.bt['data'], ApiBG.bt['timerange']) - ApiBG._bt['bt'].results = generate_backtest_stats( - ApiBG._bt['data'], ApiBG._bt['bt'].all_results, + ApiBG.bt['bt'].results = generate_backtest_stats( + ApiBG.bt['data'], ApiBG.bt['bt'].all_results, min_date=min_date, max_date=max_date) if btconfig.get('export', 'none') == 'trades': store_backtest_stats( - btconfig['exportfilename'], ApiBG._bt['bt'].results, + btconfig['exportfilename'], ApiBG.bt['bt'].results, datetime.now().strftime("%Y-%m-%d_%H-%M-%S") ) @@ -123,13 +123,13 @@ async def api_start_backtest( # noqa: C901 except (Exception, OperationalException, DependencyException) as e: logger.exception(f"Backtesting caused an error: {e}") - ApiBG._bt['bt_error'] = str(e) + ApiBG.bt['bt_error'] = str(e) pass finally: - ApiBG._bgtask_running = False + ApiBG.bgtask_running = False background_tasks.add_task(run_backtest) - ApiBG._bgtask_running = True + ApiBG.bgtask_running = True return { "status": "running", @@ -147,18 +147,18 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): Returns Result after backtesting has been ran. """ from freqtrade.persistence import LocalTrade - if ApiBG._bgtask_running: + if ApiBG.bgtask_running: return { "status": "running", "running": True, - "step": (ApiBG._bt['bt'].progress.action if ApiBG._bt['bt'] + "step": (ApiBG.bt['bt'].progress.action if ApiBG.bt['bt'] else str(BacktestState.STARTUP)), - "progress": ApiBG._bt['bt'].progress.progress if ApiBG._bt['bt'] else 0, + "progress": ApiBG.bt['bt'].progress.progress if ApiBG.bt['bt'] else 0, "trade_count": len(LocalTrade.trades), "status_msg": "Backtest running", } - if not ApiBG._bt['bt']: + if not ApiBG.bt['bt']: return { "status": "not_started", "running": False, @@ -166,13 +166,13 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest not yet executed" } - if ApiBG._bt['bt_error']: + if ApiBG.bt['bt_error']: return { "status": "error", "running": False, "step": "", "progress": 0, - "status_msg": f"Backtest failed with {ApiBG._bt['bt_error']}" + "status_msg": f"Backtest failed with {ApiBG.bt['bt_error']}" } return { @@ -181,14 +181,14 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): "status_msg": "Backtest ended", "step": "finished", "progress": 1, - "backtest_result": ApiBG._bt['bt'].results, + "backtest_result": ApiBG.bt['bt'].results, } @router.delete('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest']) def api_delete_backtest(ws_mode=Depends(is_webserver_mode)): """Reset backtesting""" - if ApiBG._bgtask_running: + if ApiBG.bgtask_running: return { "status": "running", "running": True, @@ -196,12 +196,12 @@ def api_delete_backtest(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest running", } - if ApiBG._bt['bt']: - ApiBG._bt['bt'].cleanup() - del ApiBG._bt['bt'] - ApiBG._bt['bt'] = None - del ApiBG._bt['data'] - ApiBG._bt['data'] = None + if ApiBG.bt['bt']: + ApiBG.bt['bt'].cleanup() + del ApiBG.bt['bt'] + ApiBG.bt['bt'] = None + del ApiBG.bt['data'] + ApiBG.bt['data'] = None logger.info("Backtesting reset") return { "status": "reset", @@ -214,7 +214,7 @@ def api_delete_backtest(ws_mode=Depends(is_webserver_mode)): @router.get('/backtest/abort', response_model=BacktestResponse, tags=['webserver', 'backtest']) def api_backtest_abort(ws_mode=Depends(is_webserver_mode)): - if not ApiBG._bgtask_running: + if not ApiBG.bgtask_running: return { "status": "not_running", "running": False, @@ -222,7 +222,7 @@ def api_backtest_abort(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest ended", } - ApiBG._bt['bt'].abort = True + ApiBG.bt['bt'].abort = True return { "status": "stopping", "running": False, diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index b46ac88cd..8fd105d3e 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -44,11 +44,11 @@ def get_api_config() -> Dict[str, Any]: def get_exchange(config=Depends(get_config)): - if not ApiBG._exchange: + if not ApiBG.exchange: from freqtrade.resolvers import ExchangeResolver - ApiBG._exchange = ExchangeResolver.load_exchange( + ApiBG.exchange = ExchangeResolver.load_exchange( config, load_leverage_tiers=False) - return ApiBG._exchange + return ApiBG.exchange def get_message_stream(): diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index cdb1bd031..925f34de3 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -4,13 +4,13 @@ from typing import Any, Dict class ApiBG(): # Backtesting type: Backtesting - _bt: Dict[str, Any] = { + bt: Dict[str, Any] = { 'bt': None, 'data': None, 'timerange': None, 'last_config': {}, 'bt_error': None, } - _bgtask_running: bool = False + bgtask_running: bool = False # Exchange - only available in webserver mode. - _exchange = None + exchange = None diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index aad4bedb6..3b1a2616f 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1734,7 +1734,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): assert result['status_msg'] == 'Backtest ended' # Simulate running backtest - ApiBG._bgtask_running = True + ApiBG.bgtask_running = True rc = client_get(client, f"{BASE_URI}/backtest/abort") assert_response(rc) result = rc.json() @@ -1763,7 +1763,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): result = rc.json() assert 'Bot Background task already running' in result['error'] - ApiBG._bgtask_running = False + ApiBG.bgtask_running = False # Rerun backtest (should get previous result) rc = client_post(client, f"{BASE_URI}/backtest", data=data) From 914195acf4e8ffafb3d6854039ef5ede949bf33f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:14:00 +0200 Subject: [PATCH 35/56] Ensure one test can't fail 20 others --- tests/rpc/test_rpc_apiserver.py | 224 ++++++++++++++++---------------- 1 file changed, 114 insertions(+), 110 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 3b1a2616f..842981ad0 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -21,6 +21,7 @@ from freqtrade.__init__ import __version__ from freqtrade.enums import CandleType, RunMode, State, TradingMode from freqtrade.exceptions import DependencyException, ExchangeError, OperationalException from freqtrade.loggers import setup_logging, setup_logging_pre +from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC from freqtrade.rpc.api_server import ApiServer @@ -1666,137 +1667,140 @@ def test_sysinfo(botclient): def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): - ftbot, client = botclient - mocker.patch(f'{EXMS}.get_fee', fee) + try: + ftbot, client = botclient + mocker.patch(f'{EXMS}.get_fee', fee) - rc = client_get(client, f"{BASE_URI}/backtest") - # Backtest prevented in default mode - assert_response(rc, 502) + rc = client_get(client, f"{BASE_URI}/backtest") + # Backtest prevented in default mode + assert_response(rc, 502) - ftbot.config['runmode'] = RunMode.WEBSERVER - # Backtesting not started yet - rc = client_get(client, f"{BASE_URI}/backtest") - assert_response(rc) + ftbot.config['runmode'] = RunMode.WEBSERVER + # Backtesting not started yet + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) - result = rc.json() - assert result['status'] == 'not_started' - assert not result['running'] - assert result['status_msg'] == 'Backtest not yet executed' - assert result['progress'] == 0 + result = rc.json() + assert result['status'] == 'not_started' + assert not result['running'] + assert result['status_msg'] == 'Backtest not yet executed' + assert result['progress'] == 0 - # Reset backtesting - rc = client_delete(client, f"{BASE_URI}/backtest") - assert_response(rc) - result = rc.json() - assert result['status'] == 'reset' - assert not result['running'] - assert result['status_msg'] == 'Backtest reset' - ftbot.config['export'] = 'trades' - ftbot.config['backtest_cache'] = 'day' - ftbot.config['user_data_dir'] = Path(tmpdir) - ftbot.config['exportfilename'] = Path(tmpdir) / "backtest_results" - ftbot.config['exportfilename'].mkdir() + # Reset backtesting + rc = client_delete(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'reset' + assert not result['running'] + assert result['status_msg'] == 'Backtest reset' + ftbot.config['export'] = 'trades' + ftbot.config['backtest_cache'] = 'day' + ftbot.config['user_data_dir'] = Path(tmpdir) + ftbot.config['exportfilename'] = Path(tmpdir) / "backtest_results" + ftbot.config['exportfilename'].mkdir() - # start backtesting - data = { - "strategy": CURRENT_TEST_STRATEGY, - "timeframe": "5m", - "timerange": "20180110-20180111", - "max_open_trades": 3, - "stake_amount": 100, - "dry_run_wallet": 1000, - "enable_protections": False - } - rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert_response(rc) - result = rc.json() + # start backtesting + data = { + "strategy": CURRENT_TEST_STRATEGY, + "timeframe": "5m", + "timerange": "20180110-20180111", + "max_open_trades": 3, + "stake_amount": 100, + "dry_run_wallet": 1000, + "enable_protections": False + } + rc = client_post(client, f"{BASE_URI}/backtest", data=data) + assert_response(rc) + result = rc.json() - assert result['status'] == 'running' - assert result['progress'] == 0 - assert result['running'] - assert result['status_msg'] == 'Backtest started' + assert result['status'] == 'running' + assert result['progress'] == 0 + assert result['running'] + assert result['status_msg'] == 'Backtest started' - rc = client_get(client, f"{BASE_URI}/backtest") - assert_response(rc) + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) - result = rc.json() - assert result['status'] == 'ended' - assert not result['running'] - assert result['status_msg'] == 'Backtest ended' - assert result['progress'] == 1 - assert result['backtest_result'] + result = rc.json() + assert result['status'] == 'ended' + assert not result['running'] + assert result['status_msg'] == 'Backtest ended' + assert result['progress'] == 1 + assert result['backtest_result'] - rc = client_get(client, f"{BASE_URI}/backtest/abort") - assert_response(rc) - result = rc.json() - assert result['status'] == 'not_running' - assert not result['running'] - assert result['status_msg'] == 'Backtest ended' + rc = client_get(client, f"{BASE_URI}/backtest/abort") + assert_response(rc) + result = rc.json() + assert result['status'] == 'not_running' + assert not result['running'] + assert result['status_msg'] == 'Backtest ended' - # Simulate running backtest - ApiBG.bgtask_running = True - rc = client_get(client, f"{BASE_URI}/backtest/abort") - assert_response(rc) - result = rc.json() - assert result['status'] == 'stopping' - assert not result['running'] - assert result['status_msg'] == 'Backtest ended' + # Simulate running backtest + ApiBG.bgtask_running = True + rc = client_get(client, f"{BASE_URI}/backtest/abort") + assert_response(rc) + result = rc.json() + assert result['status'] == 'stopping' + assert not result['running'] + assert result['status_msg'] == 'Backtest ended' - # Get running backtest... - rc = client_get(client, f"{BASE_URI}/backtest") - assert_response(rc) - result = rc.json() - assert result['status'] == 'running' - assert result['running'] - assert result['step'] == "backtest" - assert result['status_msg'] == "Backtest running" + # Get running backtest... + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'running' + assert result['running'] + assert result['step'] == "backtest" + assert result['status_msg'] == "Backtest running" - # Try delete with task still running - rc = client_delete(client, f"{BASE_URI}/backtest") - assert_response(rc) - result = rc.json() - assert result['status'] == 'running' + # Try delete with task still running + rc = client_delete(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'running' - # Post to backtest that's still running - rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert_response(rc, 502) - result = rc.json() - assert 'Bot Background task already running' in result['error'] + # Post to backtest that's still running + rc = client_post(client, f"{BASE_URI}/backtest", data=data) + assert_response(rc, 502) + result = rc.json() + assert 'Bot Background task already running' in result['error'] - ApiBG.bgtask_running = False + ApiBG.bgtask_running = False - # Rerun backtest (should get previous result) - rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert_response(rc) - result = rc.json() - assert log_has_re('Reusing result of previous backtest.*', caplog) + # Rerun backtest (should get previous result) + rc = client_post(client, f"{BASE_URI}/backtest", data=data) + assert_response(rc) + result = rc.json() + assert log_has_re('Reusing result of previous backtest.*', caplog) - data['stake_amount'] = 101 + data['stake_amount'] = 101 - mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', - side_effect=DependencyException('DeadBeef')) - rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert log_has("Backtesting caused an error: DeadBeef", caplog) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', + side_effect=DependencyException('DeadBeef')) + rc = client_post(client, f"{BASE_URI}/backtest", data=data) + assert log_has("Backtesting caused an error: DeadBeef", caplog) - rc = client_get(client, f"{BASE_URI}/backtest") - assert_response(rc) - result = rc.json() - assert result['status'] == 'error' - assert 'Backtest failed' in result['status_msg'] + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'error' + assert 'Backtest failed' in result['status_msg'] - # Delete backtesting to avoid leakage since the backtest-object may stick around. - rc = client_delete(client, f"{BASE_URI}/backtest") - assert_response(rc) + # Delete backtesting to avoid leakage since the backtest-object may stick around. + rc = client_delete(client, f"{BASE_URI}/backtest") + assert_response(rc) - result = rc.json() - assert result['status'] == 'reset' - assert not result['running'] - assert result['status_msg'] == 'Backtest reset' + result = rc.json() + assert result['status'] == 'reset' + assert not result['running'] + assert result['status_msg'] == 'Backtest reset' - # Disallow base64 strategies - data['strategy'] = "xx:cHJpbnQoImhlbGxvIHdvcmxkIik=" - rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert_response(rc, 500) + # Disallow base64 strategies + data['strategy'] = "xx:cHJpbnQoImhlbGxvIHdvcmxkIik=" + rc = client_post(client, f"{BASE_URI}/backtest", data=data) + assert_response(rc, 500) + finally: + Backtesting.cleanup() def test_api_backtest_history(botclient, mocker, testdatadir): From a87b215d67bce2fdd46455824f3ba63fdff0ab15 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:50:59 +0200 Subject: [PATCH 36/56] Fix odd import --- freqtrade/freqai/torch/PyTorchTransformerModel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/torch/PyTorchTransformerModel.py b/freqtrade/freqai/torch/PyTorchTransformerModel.py index 702a7a08b..162459776 100644 --- a/freqtrade/freqai/torch/PyTorchTransformerModel.py +++ b/freqtrade/freqai/torch/PyTorchTransformerModel.py @@ -1,7 +1,7 @@ import math import torch -import torch.nn as nn +from torch import nn """ @@ -68,7 +68,7 @@ class PyTorchTransformerModel(nn.Module): return x -class PositionalEncoding(torch.nn.Module): +class PositionalEncoding(nn.Module): def __init__(self, d_model, max_len=5000): """ Args From abc82a3fdf36e522ca360fe55c2cc87de091689a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 12:04:18 +0200 Subject: [PATCH 37/56] Simplify api backtesting by extracting the background_method --- freqtrade/rpc/api_server/api_backtest.py | 148 ++++++++++++----------- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index a06d65dcc..8fa1a87b8 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -8,6 +8,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends from fastapi.exceptions import HTTPException from freqtrade.configuration.config_validation import validate_config_consistency +from freqtrade.constants import Config from freqtrade.data.btanalysis import get_backtest_resultlist, load_and_merge_backtest_result from freqtrade.enums import BacktestState from freqtrade.exceptions import DependencyException, OperationalException @@ -26,8 +27,80 @@ logger = logging.getLogger(__name__) router = APIRouter() +def __run_backtest_bg(btconfig: Config): + from freqtrade.optimize.optimize_reports import generate_backtest_stats, store_backtest_stats + from freqtrade.resolvers import StrategyResolver + asyncio.set_event_loop(asyncio.new_event_loop()) + try: + # Reload strategy + lastconfig = ApiBG.bt['last_config'] + strat = StrategyResolver.load_strategy(btconfig) + validate_config_consistency(btconfig) + + if ( + not ApiBG.bt['bt'] + or lastconfig.get('timeframe') != strat.timeframe + or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') + or lastconfig.get('timerange') != btconfig['timerange'] + ): + from freqtrade.optimize.backtesting import Backtesting + ApiBG.bt['bt'] = Backtesting(btconfig) + ApiBG.bt['bt'].load_bt_data_detail() + else: + ApiBG.bt['bt'].config = btconfig + ApiBG.bt['bt'].init_backtest() + # Only reload data if timeframe changed. + if ( + not ApiBG.bt['data'] + or not ApiBG.bt['timerange'] + or lastconfig.get('timeframe') != strat.timeframe + or lastconfig.get('timerange') != btconfig['timerange'] + ): + ApiBG.bt['data'], ApiBG.bt['timerange'] = ApiBG.bt[ + 'bt'].load_bt_data() + + lastconfig['timerange'] = btconfig['timerange'] + lastconfig['timeframe'] = strat.timeframe + lastconfig['protections'] = btconfig.get('protections', []) + lastconfig['enable_protections'] = btconfig.get('enable_protections') + lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') + + ApiBG.bt['bt'].enable_protections = btconfig.get('enable_protections', False) + ApiBG.bt['bt'].strategylist = [strat] + ApiBG.bt['bt'].results = {} + ApiBG.bt['bt'].load_prior_backtest() + + ApiBG.bt['bt'].abort = False + if (ApiBG.bt['bt'].results and + strat.get_strategy_name() in ApiBG.bt['bt'].results['strategy']): + # When previous result hash matches - reuse that result and skip backtesting. + logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}') + else: + min_date, max_date = ApiBG.bt['bt'].backtest_one_strategy( + strat, ApiBG.bt['data'], ApiBG.bt['timerange']) + + ApiBG.bt['bt'].results = generate_backtest_stats( + ApiBG.bt['data'], ApiBG.bt['bt'].all_results, + min_date=min_date, max_date=max_date) + + if btconfig.get('export', 'none') == 'trades': + store_backtest_stats( + btconfig['exportfilename'], ApiBG.bt['bt'].results, + datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + ) + + logger.info("Backtest finished.") + + except (Exception, OperationalException, DependencyException) as e: + logger.exception(f"Backtesting caused an error: {e}") + ApiBG.bt['bt_error'] = str(e) + pass + finally: + ApiBG.bgtask_running = False + + @router.post('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest']) -async def api_start_backtest( # noqa: C901 +async def api_start_backtest( bt_settings: BacktestRequest, background_tasks: BackgroundTasks, config=Depends(get_config), ws_mode=Depends(is_webserver_mode)): ApiBG.bt['bt_error'] = None @@ -56,79 +129,8 @@ async def api_start_backtest( # noqa: C901 # Start backtesting # Initialize backtesting object - def run_backtest(): - from freqtrade.optimize.optimize_reports import (generate_backtest_stats, - store_backtest_stats) - from freqtrade.resolvers import StrategyResolver - asyncio.set_event_loop(asyncio.new_event_loop()) - try: - # Reload strategy - lastconfig = ApiBG.bt['last_config'] - strat = StrategyResolver.load_strategy(btconfig) - validate_config_consistency(btconfig) - if ( - not ApiBG.bt['bt'] - or lastconfig.get('timeframe') != strat.timeframe - or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') - or lastconfig.get('timerange') != btconfig['timerange'] - ): - from freqtrade.optimize.backtesting import Backtesting - ApiBG.bt['bt'] = Backtesting(btconfig) - ApiBG.bt['bt'].load_bt_data_detail() - else: - ApiBG.bt['bt'].config = btconfig - ApiBG.bt['bt'].init_backtest() - # Only reload data if timeframe changed. - if ( - not ApiBG.bt['data'] - or not ApiBG.bt['timerange'] - or lastconfig.get('timeframe') != strat.timeframe - or lastconfig.get('timerange') != btconfig['timerange'] - ): - ApiBG.bt['data'], ApiBG.bt['timerange'] = ApiBG.bt[ - 'bt'].load_bt_data() - - lastconfig['timerange'] = btconfig['timerange'] - lastconfig['timeframe'] = strat.timeframe - lastconfig['protections'] = btconfig.get('protections', []) - lastconfig['enable_protections'] = btconfig.get('enable_protections') - lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') - - ApiBG.bt['bt'].enable_protections = btconfig.get('enable_protections', False) - ApiBG.bt['bt'].strategylist = [strat] - ApiBG.bt['bt'].results = {} - ApiBG.bt['bt'].load_prior_backtest() - - ApiBG.bt['bt'].abort = False - if (ApiBG.bt['bt'].results and - strat.get_strategy_name() in ApiBG.bt['bt'].results['strategy']): - # When previous result hash matches - reuse that result and skip backtesting. - logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}') - else: - min_date, max_date = ApiBG.bt['bt'].backtest_one_strategy( - strat, ApiBG.bt['data'], ApiBG.bt['timerange']) - - ApiBG.bt['bt'].results = generate_backtest_stats( - ApiBG.bt['data'], ApiBG.bt['bt'].all_results, - min_date=min_date, max_date=max_date) - - if btconfig.get('export', 'none') == 'trades': - store_backtest_stats( - btconfig['exportfilename'], ApiBG.bt['bt'].results, - datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - ) - - logger.info("Backtest finished.") - - except (Exception, OperationalException, DependencyException) as e: - logger.exception(f"Backtesting caused an error: {e}") - ApiBG.bt['bt_error'] = str(e) - pass - finally: - ApiBG.bgtask_running = False - - background_tasks.add_task(run_backtest) + background_tasks.add_task(__run_backtest_bg, btconfig=btconfig) ApiBG.bgtask_running = True return { From 68ab147f5765a6f8e028b1c430675d01d44abac8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 May 2023 12:52:49 +0000 Subject: [PATCH 38/56] Bump cachetools from 4.2.2 to 5.3.0 Bumps [cachetools](https://github.com/tkem/cachetools) from 4.2.2 to 5.3.0. - [Release notes](https://github.com/tkem/cachetools/releases) - [Changelog](https://github.com/tkem/cachetools/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tkem/cachetools/compare/v4.2.2...v5.3.0) --- updated-dependencies: - dependency-name: cachetools dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5d2f4147c..4c5044877 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ python-telegram-bot==20.3 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.23.3 arrow==1.2.3 -cachetools==4.2.2 +cachetools==5.3.0 requests==2.30.0 urllib3==2.0.2 jsonschema==4.17.3 From 8c866abad8fdb94ccfdf364957984627789b8b75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 03:56:52 +0000 Subject: [PATCH 39/56] Bump ccxt from 3.0.103 to 3.1.5 Bumps [ccxt](https://github.com/ccxt/ccxt) from 3.0.103 to 3.1.5. - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/3.0.103...3.1.5) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4c5044877..86dfc0dee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.24.3 pandas==2.0.1 pandas-ta==0.3.14b -ccxt==3.0.103 +ccxt==3.1.5 cryptography==40.0.2; platform_machine != 'armv7l' cryptography==40.0.1; platform_machine == 'armv7l' aiohttp==3.8.4 From 4e3de64c573f113e5a4d8cd4bfdd8e97adc461e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 03:57:01 +0000 Subject: [PATCH 40/56] Bump fastapi from 0.95.1 to 0.95.2 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.95.1 to 0.95.2. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.95.1...0.95.2) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4c5044877..bb894803e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ orjson==3.8.12 sdnotify==0.3.2 # API Server -fastapi==0.95.1 +fastapi==0.95.2 pydantic==1.10.7 uvicorn==0.22.0 pyjwt==2.7.0 From 811198cc3e87ecc038d9ff25341cb6d9f0e42fea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 03:57:15 +0000 Subject: [PATCH 41/56] Bump pre-commit from 3.3.1 to 3.3.2 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 3.3.1 to 3.3.2. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v3.3.1...v3.3.2) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b1ba32b43..fdb9dfde7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,7 +9,7 @@ coveralls==3.3.1 ruff==0.0.267 mypy==1.3.0 -pre-commit==3.3.1 +pre-commit==3.3.2 pytest==7.3.1 pytest-asyncio==0.21.0 pytest-cov==4.0.0 From ae52ce8cdaae988d0e9d7f9cedfb4c6fea64832f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 03:58:23 +0000 Subject: [PATCH 42/56] Bump sqlalchemy from 2.0.13 to 2.0.15 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 2.0.13 to 2.0.15. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4c5044877..a19327b97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==3.0.103 cryptography==40.0.2; platform_machine != 'armv7l' cryptography==40.0.1; platform_machine == 'armv7l' aiohttp==3.8.4 -SQLAlchemy==2.0.13 +SQLAlchemy==2.0.15 python-telegram-bot==20.3 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.23.3 From 96eb109b4edfa63f02148fd53f8fbd94b23bc90c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 03:59:01 +0000 Subject: [PATCH 43/56] Bump mkdocs-material from 9.1.12 to 9.1.14 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.1.12 to 9.1.14. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.1.12...9.1.14) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index f7c0aebe9..c5e478c78 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.4.3 -mkdocs-material==9.1.12 +mkdocs-material==9.1.14 mdx_truly_sane_lists==1.3 pymdown-extensions==10.0.1 jinja2==3.1.2 From ad5f0307b7c3c40c472f9acee37c6d2cb98a469d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 May 2023 09:21:18 +0200 Subject: [PATCH 44/56] bump sqlalchemy precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a85c84eb8..4be298d7b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - types-requests==2.30.0.0 - types-tabulate==0.9.0.2 - types-python-dateutil==2.8.19.13 - - SQLAlchemy==2.0.13 + - SQLAlchemy==2.0.15 # stages: [push] - repo: https://github.com/pycqa/isort From 2242d544fcb2e6df6082bbb8aa4e884a533427a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 07:21:19 +0000 Subject: [PATCH 45/56] Bump ruff from 0.0.267 to 0.0.269 Bumps [ruff](https://github.com/charliermarsh/ruff) from 0.0.267 to 0.0.269. - [Release notes](https://github.com/charliermarsh/ruff/releases) - [Changelog](https://github.com/charliermarsh/ruff/blob/main/BREAKING_CHANGES.md) - [Commits](https://github.com/charliermarsh/ruff/compare/v0.0.267...v0.0.269) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index fdb9dfde7..cc3463174 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.0.267 +ruff==0.0.269 mypy==1.3.0 pre-commit==3.3.2 pytest==7.3.1 From d26aa231fc8eb20a582945df6d9a1522af2cc8ce Mon Sep 17 00:00:00 2001 From: Richard Jozsa <38407205+richardjozsa@users.noreply.github.com> Date: Mon, 22 May 2023 10:36:07 +0200 Subject: [PATCH 46/56] Stable baselines updates, and fix There was a seeding error in SB3 after the gymnasium update, the stable baselines team has patched and fixed the issue, but the reset function has to be aligned. --- freqtrade/freqai/RL/BaseEnvironment.py | 2 +- requirements-freqai-rl.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqai/RL/BaseEnvironment.py b/freqtrade/freqai/RL/BaseEnvironment.py index 7c83a7e42..42e644f0a 100644 --- a/freqtrade/freqai/RL/BaseEnvironment.py +++ b/freqtrade/freqai/RL/BaseEnvironment.py @@ -180,7 +180,7 @@ class BaseEnvironment(gym.Env): def reset_tensorboard_log(self): self.tensorboard_metrics = {} - def reset(self): + def reset(self, seed=None): """ Reset is called at the beginning of every episode """ diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index 535d10f4b..0e9df61f1 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -5,7 +5,7 @@ torch==2.0.1 #until these branches will be released we can use this gymnasium==0.28.1 -stable_baselines3==2.0.0a5 -sb3_contrib>=2.0.0a4 +stable_baselines3==2.0.0a9 +sb3_contrib>=2.0.0a9 # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.65.0 From 44bdac5e8c81a4ebf96e0d8a6119e5d1134c55ad Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 May 2023 18:23:33 +0200 Subject: [PATCH 47/56] Improve developer docs with some minor improvements --- docs/developer.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 1bc75551f..2782f0117 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -327,18 +327,18 @@ To check how the new exchange behaves, you can use the following snippet: ``` python import ccxt -from datetime import datetime +from datetime import datetime, timezone from freqtrade.data.converter import ohlcv_to_dataframe -ct = ccxt.binance() +ct = ccxt.binance() # Use the exchange you're testing timeframe = "1d" -pair = "XLM/BTC" # Make sure to use a pair that exists on that exchange! +pair = "BTC/USDT" # Make sure to use a pair that exists on that exchange! raw = ct.fetch_ohlcv(pair, timeframe=timeframe) # convert to dataframe df1 = ohlcv_to_dataframe(raw, timeframe, pair=pair, drop_incomplete=False) print(df1.tail(1)) -print(datetime.utcnow()) +print(datetime.now(timezone.utc)) ``` ``` output From 1e10b25e3da3768a97780d6919fbb985b415e6f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 21:58:08 +0000 Subject: [PATCH 48/56] Bump requests from 2.30.0 to 2.31.0 Bumps [requests](https://github.com/psf/requests) from 2.30.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.30.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 48ecb4d34..cff54ca1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ python-telegram-bot==20.3 httpx>=0.23.3 arrow==1.2.3 cachetools==5.3.0 -requests==2.30.0 +requests==2.31.0 urllib3==2.0.2 jsonschema==4.17.3 TA-Lib==0.4.26 From 9ffdaceef3bbac11cff7078791c7667e3f513d73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 May 2023 07:01:48 +0200 Subject: [PATCH 49/56] Bybit - use Proxy --- tests/exchange/test_ccxt_compat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 6f5987202..0561d5017 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -226,6 +226,7 @@ EXCHANGES = { 'pair': 'BTC/USDT', 'stake_currency': 'USDT', 'hasQuoteVolume': True, + 'use_ci_proxy': True, 'timeframe': '1h', 'futures_pair': 'BTC/USDT:USDT', 'futures': True, From 6292d1af6d30694c0862128c81f561537bf0bc10 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 May 2023 19:07:58 +0200 Subject: [PATCH 50/56] Use camelcase version of private fapi method closes #8680 --- freqtrade/exchange/binance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index caca3eefb..8075d775a 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -65,7 +65,7 @@ class Binance(Exchange): """ try: if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']: - position_side = self._api.fapiPrivateGetPositionsideDual() + position_side = self._api.fapiPrivateGetPositionSideDual() self._log_exchange_response('position_side_setting', position_side) assets_margin = self._api.fapiPrivateGetMultiAssetsMargin() self._log_exchange_response('multi_asset_margin', assets_margin) From 6efc62e4cd34e263e603d1446bc5bd1d0983f405 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 May 2023 19:10:10 +0200 Subject: [PATCH 51/56] Add test which verifies #8680 won't happen again --- tests/exchange/test_ccxt_compat.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 0561d5017..404b51d10 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -43,6 +43,10 @@ EXCHANGES = { 'hasQuoteVolumeFutures': True, 'leverage_tiers_public': False, 'leverage_in_spot_market': False, + 'private_methods': [ + 'fapiPrivateGetPositionSideDual', + 'fapiPrivateGetMultiAssetsMargin' + ], 'sample_order': [{ "symbol": "SOLUSDT", "orderId": 3551312894, @@ -221,6 +225,7 @@ EXCHANGES = { 'hasQuoteVolumeFutures': False, 'leverage_tiers_public': True, 'leverage_in_spot_market': True, + 'private_methods': ['fetch_accounts'], }, 'bybit': { 'pair': 'BTC/USDT', @@ -756,3 +761,8 @@ class TestCCXTExchange(): max_stake_amount = futures.get_max_pair_stake_amount(futures_pair, 40000) assert (isinstance(max_stake_amount, float)) assert max_stake_amount >= 0.0 + + def test_private_method_presence(self, exchange: EXCHANGE_FIXTURE_TYPE): + exch, exchangename = exchange + for method in EXCHANGES[exchangename].get('private_methods', []): + assert hasattr(exch._api, method) From a0336c83c3564dfd6b5cfcacc5534aebffd13184 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 May 2023 19:22:58 +0200 Subject: [PATCH 52/56] Update method casing in tests --- tests/exchange/test_binance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index d44dae00d..9018d2db9 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -514,7 +514,7 @@ def test_fill_leverage_tiers_binance_dryrun(default_conf, mocker, leverage_tiers def test_additional_exchange_init_binance(default_conf, mocker): api_mock = MagicMock() - api_mock.fapiPrivateGetPositionsideDual = MagicMock(return_value={"dualSidePosition": True}) + api_mock.fapiPrivateGetPositionSideDual = MagicMock(return_value={"dualSidePosition": True}) api_mock.fapiPrivateGetMultiAssetsMargin = MagicMock(return_value={"multiAssetsMargin": True}) default_conf['dry_run'] = False default_conf['trading_mode'] = TradingMode.FUTURES @@ -522,12 +522,12 @@ def test_additional_exchange_init_binance(default_conf, mocker): with pytest.raises(OperationalException, match=r"Hedge Mode is not supported.*\nMulti-Asset Mode is not supported.*"): get_patched_exchange(mocker, default_conf, id="binance", api_mock=api_mock) - api_mock.fapiPrivateGetPositionsideDual = MagicMock(return_value={"dualSidePosition": False}) + api_mock.fapiPrivateGetPositionSideDual = MagicMock(return_value={"dualSidePosition": False}) api_mock.fapiPrivateGetMultiAssetsMargin = MagicMock(return_value={"multiAssetsMargin": False}) exchange = get_patched_exchange(mocker, default_conf, id="binance", api_mock=api_mock) assert exchange ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'binance', - "additional_exchange_init", "fapiPrivateGetPositionsideDual") + "additional_exchange_init", "fapiPrivateGetPositionSideDual") def test__set_leverage_binance(mocker, default_conf): From b8220ee0f7b0d8fd2495cd33cc1763b07a6be540 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 May 2023 18:19:14 +0200 Subject: [PATCH 53/56] Improve recovery detection by skipping open orders --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 21426623f..fc4c65caf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1075,7 +1075,7 @@ class FreqtradeBot(LoggingMixin): trades_closed = 0 for trade in trades: - if not self.wallets.check_exit_amount(trade): + if trade.open_order_id is None and not self.wallets.check_exit_amount(trade): logger.warning( f'Not enough {trade.safe_base_currency} in wallet to exit {trade}. ' 'Trying to recover.') From b5ed693bee93176cf3107fc4f06f98e64693a552 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 May 2023 20:14:16 +0200 Subject: [PATCH 54/56] Extrac OKX convert stop order, call for regular orders, too --- freqtrade/exchange/okx.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 84b7deb7a..af889897c 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -169,6 +169,22 @@ class Okx(Exchange): params['posSide'] = self._get_posSide(side, True) return params + def _convert_stop_order(self, pair: str, order_id: str, order: Dict) -> Dict: + if ( + order['status'] == 'closed' + and (real_order_id := order.get('info', {}).get('ordId')) is not None + ): + # Once a order triggered, we fetch the regular followup order. + order_reg = self.fetch_order(real_order_id, pair) + self._log_exchange_response('fetch_stoploss_order1', order_reg) + order_reg['id_stop'] = order_reg['id'] + order_reg['id'] = order_id + order_reg['type'] = 'stoploss' + order_reg['status_stop'] = 'triggered' + return order_reg + order['type'] = 'stoploss' + return order + def fetch_stoploss_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict: if self._config['dry_run']: return self.fetch_dry_run_order(order_id) @@ -177,7 +193,7 @@ class Okx(Exchange): params1 = {'stop': True} order_reg = self._api.fetch_order(order_id, pair, params=params1) self._log_exchange_response('fetch_stoploss_order', order_reg) - return order_reg + return self._convert_stop_order(pair, order_id, order_reg) except ccxt.OrderNotFound: pass params2 = {'stop': True, 'ordType': 'conditional'} @@ -188,18 +204,7 @@ class Okx(Exchange): orders_f = [order for order in orders if order['id'] == order_id] if orders_f: order = orders_f[0] - if (order['status'] == 'closed' - and (real_order_id := order.get('info', {}).get('ordId')) is not None): - # Once a order triggered, we fetch the regular followup order. - order_reg = self.fetch_order(real_order_id, pair) - self._log_exchange_response('fetch_stoploss_order1', order_reg) - order_reg['id_stop'] = order_reg['id'] - order_reg['id'] = order_id - order_reg['type'] = 'stoploss' - order_reg['status_stop'] = 'triggered' - return order_reg - order['type'] = 'stoploss' - return order + return self._convert_stop_order(pair, order_id, order) except ccxt.BaseError: pass raise RetryableOrderError( From 9e9f9b21e5a833a3213499950d25e6c809ebfd59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 04:24:42 +0000 Subject: [PATCH 55/56] Bump stable-baselines3 from 2.0.0a9 to 2.0.0a10 Bumps [stable-baselines3](https://github.com/DLR-RM/stable-baselines3) from 2.0.0a9 to 2.0.0a10. - [Release notes](https://github.com/DLR-RM/stable-baselines3/releases) - [Commits](https://github.com/DLR-RM/stable-baselines3/commits) --- updated-dependencies: - dependency-name: stable-baselines3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-freqai-rl.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index 0e9df61f1..de48a1da4 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -5,7 +5,7 @@ torch==2.0.1 #until these branches will be released we can use this gymnasium==0.28.1 -stable_baselines3==2.0.0a9 +stable_baselines3==2.0.0a10 sb3_contrib>=2.0.0a9 # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.65.0 From f647fb342bc8ac2dc1ae33001cb8e5a39009a911 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Thu, 25 May 2023 16:35:06 +0200 Subject: [PATCH 56/56] Update base_tensorboard.py Remove incorrect warning message. --- freqtrade/freqai/tensorboard/base_tensorboard.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/tensorboard/base_tensorboard.py b/freqtrade/freqai/tensorboard/base_tensorboard.py index c2d47137e..72f47111c 100644 --- a/freqtrade/freqai/tensorboard/base_tensorboard.py +++ b/freqtrade/freqai/tensorboard/base_tensorboard.py @@ -10,8 +10,7 @@ logger = logging.getLogger(__name__) class BaseTensorboardLogger: def __init__(self, logdir: Path, activate: bool = True): - logger.warning("Tensorboard is not installed, no logs will be written." - "Ensure torch is installed, or use the torch/RL docker images") + pass def log_scalar(self, tag: str, scalar_value: Any, step: int): return @@ -23,8 +22,7 @@ class BaseTensorboardLogger: class BaseTensorBoardCallback(TrainingCallback): def __init__(self, logdir: Path, activate: bool = True): - logger.warning("Tensorboard is not installed, no logs will be written." - "Ensure torch is installed, or use the torch/RL docker images") + pass def after_iteration( self, model, epoch: int, evals_log: TrainingCallback.EvalsLog