From 97a6fb285f894ba91b40dc8fa51d28d750d894b0 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 10 Jan 2023 17:52:24 -0700 Subject: [PATCH 01/17] revert to dataframe.to_json --- freqtrade/misc.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 93e8da6dd..7b9ff1f1d 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -10,7 +10,6 @@ from typing import Any, Dict, Iterator, List, Mapping, Union from typing.io import IO from urllib.parse import urlparse -import orjson import pandas as pd import rapidjson @@ -263,15 +262,7 @@ def dataframe_to_json(dataframe: pd.DataFrame) -> str: :param dataframe: A pandas DataFrame :returns: A JSON string of the pandas DataFrame """ - # https://github.com/pandas-dev/pandas/issues/24889 - # https://github.com/pandas-dev/pandas/issues/40443 - # We need to convert to a dict to avoid mem leak - def default(z): - if isinstance(z, pd.Timestamp): - return z.timestamp() * 1e3 - raise TypeError - - return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8') + return dataframe.to_json(orient='split') def json_to_dataframe(data: str) -> pd.DataFrame: From 3a7e41e1777dc4835b7d1e3ed35318d829f88844 Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Wed, 10 May 2023 10:32:00 +0100 Subject: [PATCH 02/17] Update rest_client.py Add fix for forceenter to avoid passing None prices back to the API --- scripts/rest_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ccffe7f5f..d301d06e6 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -279,8 +279,9 @@ class FtRestClient(): """ data = {"pair": pair, "side": side, - "price": price, } + if price: + params['price'] = price return self._post("forceenter", data=data) def forceexit(self, tradeid, ordertype=None, amount=None): From 242247be47a629efef0d412e8b27a28e5aae944f Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Wed, 10 May 2023 10:56:14 +0100 Subject: [PATCH 03/17] Fix var name --- scripts/rest_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rest_client.py b/scripts/rest_client.py index d301d06e6..0772af269 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -281,7 +281,7 @@ class FtRestClient(): "side": side, } if price: - params['price'] = price + data['price'] = price return self._post("forceenter", data=data) def forceexit(self, tradeid, ordertype=None, amount=None): From d26aa231fc8eb20a582945df6d9a1522af2cc8ce Mon Sep 17 00:00:00 2001 From: Richard Jozsa <38407205+richardjozsa@users.noreply.github.com> Date: Mon, 22 May 2023 10:36:07 +0200 Subject: [PATCH 04/17] Stable baselines updates, and fix There was a seeding error in SB3 after the gymnasium update, the stable baselines team has patched and fixed the issue, but the reset function has to be aligned. --- freqtrade/freqai/RL/BaseEnvironment.py | 2 +- requirements-freqai-rl.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqai/RL/BaseEnvironment.py b/freqtrade/freqai/RL/BaseEnvironment.py index 7c83a7e42..42e644f0a 100644 --- a/freqtrade/freqai/RL/BaseEnvironment.py +++ b/freqtrade/freqai/RL/BaseEnvironment.py @@ -180,7 +180,7 @@ class BaseEnvironment(gym.Env): def reset_tensorboard_log(self): self.tensorboard_metrics = {} - def reset(self): + def reset(self, seed=None): """ Reset is called at the beginning of every episode """ diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index 535d10f4b..0e9df61f1 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -5,7 +5,7 @@ torch==2.0.1 #until these branches will be released we can use this gymnasium==0.28.1 -stable_baselines3==2.0.0a5 -sb3_contrib>=2.0.0a4 +stable_baselines3==2.0.0a9 +sb3_contrib>=2.0.0a9 # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.65.0 From 1e10b25e3da3768a97780d6919fbb985b415e6f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 21:58:08 +0000 Subject: [PATCH 05/17] Bump requests from 2.30.0 to 2.31.0 Bumps [requests](https://github.com/psf/requests) from 2.30.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.30.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 48ecb4d34..cff54ca1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ python-telegram-bot==20.3 httpx>=0.23.3 arrow==1.2.3 cachetools==5.3.0 -requests==2.30.0 +requests==2.31.0 urllib3==2.0.2 jsonschema==4.17.3 TA-Lib==0.4.26 From 9ffdaceef3bbac11cff7078791c7667e3f513d73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 May 2023 07:01:48 +0200 Subject: [PATCH 06/17] Bybit - use Proxy --- tests/exchange/test_ccxt_compat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 6f5987202..0561d5017 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -226,6 +226,7 @@ EXCHANGES = { 'pair': 'BTC/USDT', 'stake_currency': 'USDT', 'hasQuoteVolume': True, + 'use_ci_proxy': True, 'timeframe': '1h', 'futures_pair': 'BTC/USDT:USDT', 'futures': True, From 6292d1af6d30694c0862128c81f561537bf0bc10 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 May 2023 19:07:58 +0200 Subject: [PATCH 07/17] Use camelcase version of private fapi method closes #8680 --- freqtrade/exchange/binance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index caca3eefb..8075d775a 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -65,7 +65,7 @@ class Binance(Exchange): """ try: if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']: - position_side = self._api.fapiPrivateGetPositionsideDual() + position_side = self._api.fapiPrivateGetPositionSideDual() self._log_exchange_response('position_side_setting', position_side) assets_margin = self._api.fapiPrivateGetMultiAssetsMargin() self._log_exchange_response('multi_asset_margin', assets_margin) From 6efc62e4cd34e263e603d1446bc5bd1d0983f405 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 May 2023 19:10:10 +0200 Subject: [PATCH 08/17] Add test which verifies #8680 won't happen again --- tests/exchange/test_ccxt_compat.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 0561d5017..404b51d10 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -43,6 +43,10 @@ EXCHANGES = { 'hasQuoteVolumeFutures': True, 'leverage_tiers_public': False, 'leverage_in_spot_market': False, + 'private_methods': [ + 'fapiPrivateGetPositionSideDual', + 'fapiPrivateGetMultiAssetsMargin' + ], 'sample_order': [{ "symbol": "SOLUSDT", "orderId": 3551312894, @@ -221,6 +225,7 @@ EXCHANGES = { 'hasQuoteVolumeFutures': False, 'leverage_tiers_public': True, 'leverage_in_spot_market': True, + 'private_methods': ['fetch_accounts'], }, 'bybit': { 'pair': 'BTC/USDT', @@ -756,3 +761,8 @@ class TestCCXTExchange(): max_stake_amount = futures.get_max_pair_stake_amount(futures_pair, 40000) assert (isinstance(max_stake_amount, float)) assert max_stake_amount >= 0.0 + + def test_private_method_presence(self, exchange: EXCHANGE_FIXTURE_TYPE): + exch, exchangename = exchange + for method in EXCHANGES[exchangename].get('private_methods', []): + assert hasattr(exch._api, method) From a0336c83c3564dfd6b5cfcacc5534aebffd13184 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 May 2023 19:22:58 +0200 Subject: [PATCH 09/17] Update method casing in tests --- tests/exchange/test_binance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index d44dae00d..9018d2db9 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -514,7 +514,7 @@ def test_fill_leverage_tiers_binance_dryrun(default_conf, mocker, leverage_tiers def test_additional_exchange_init_binance(default_conf, mocker): api_mock = MagicMock() - api_mock.fapiPrivateGetPositionsideDual = MagicMock(return_value={"dualSidePosition": True}) + api_mock.fapiPrivateGetPositionSideDual = MagicMock(return_value={"dualSidePosition": True}) api_mock.fapiPrivateGetMultiAssetsMargin = MagicMock(return_value={"multiAssetsMargin": True}) default_conf['dry_run'] = False default_conf['trading_mode'] = TradingMode.FUTURES @@ -522,12 +522,12 @@ def test_additional_exchange_init_binance(default_conf, mocker): with pytest.raises(OperationalException, match=r"Hedge Mode is not supported.*\nMulti-Asset Mode is not supported.*"): get_patched_exchange(mocker, default_conf, id="binance", api_mock=api_mock) - api_mock.fapiPrivateGetPositionsideDual = MagicMock(return_value={"dualSidePosition": False}) + api_mock.fapiPrivateGetPositionSideDual = MagicMock(return_value={"dualSidePosition": False}) api_mock.fapiPrivateGetMultiAssetsMargin = MagicMock(return_value={"multiAssetsMargin": False}) exchange = get_patched_exchange(mocker, default_conf, id="binance", api_mock=api_mock) assert exchange ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'binance', - "additional_exchange_init", "fapiPrivateGetPositionsideDual") + "additional_exchange_init", "fapiPrivateGetPositionSideDual") def test__set_leverage_binance(mocker, default_conf): From b8220ee0f7b0d8fd2495cd33cc1763b07a6be540 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 May 2023 18:19:14 +0200 Subject: [PATCH 10/17] Improve recovery detection by skipping open orders --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 21426623f..fc4c65caf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1075,7 +1075,7 @@ class FreqtradeBot(LoggingMixin): trades_closed = 0 for trade in trades: - if not self.wallets.check_exit_amount(trade): + if trade.open_order_id is None and not self.wallets.check_exit_amount(trade): logger.warning( f'Not enough {trade.safe_base_currency} in wallet to exit {trade}. ' 'Trying to recover.') From b5ed693bee93176cf3107fc4f06f98e64693a552 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 May 2023 20:14:16 +0200 Subject: [PATCH 11/17] Extrac OKX convert stop order, call for regular orders, too --- freqtrade/exchange/okx.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 84b7deb7a..af889897c 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -169,6 +169,22 @@ class Okx(Exchange): params['posSide'] = self._get_posSide(side, True) return params + def _convert_stop_order(self, pair: str, order_id: str, order: Dict) -> Dict: + if ( + order['status'] == 'closed' + and (real_order_id := order.get('info', {}).get('ordId')) is not None + ): + # Once a order triggered, we fetch the regular followup order. + order_reg = self.fetch_order(real_order_id, pair) + self._log_exchange_response('fetch_stoploss_order1', order_reg) + order_reg['id_stop'] = order_reg['id'] + order_reg['id'] = order_id + order_reg['type'] = 'stoploss' + order_reg['status_stop'] = 'triggered' + return order_reg + order['type'] = 'stoploss' + return order + def fetch_stoploss_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict: if self._config['dry_run']: return self.fetch_dry_run_order(order_id) @@ -177,7 +193,7 @@ class Okx(Exchange): params1 = {'stop': True} order_reg = self._api.fetch_order(order_id, pair, params=params1) self._log_exchange_response('fetch_stoploss_order', order_reg) - return order_reg + return self._convert_stop_order(pair, order_id, order_reg) except ccxt.OrderNotFound: pass params2 = {'stop': True, 'ordType': 'conditional'} @@ -188,18 +204,7 @@ class Okx(Exchange): orders_f = [order for order in orders if order['id'] == order_id] if orders_f: order = orders_f[0] - if (order['status'] == 'closed' - and (real_order_id := order.get('info', {}).get('ordId')) is not None): - # Once a order triggered, we fetch the regular followup order. - order_reg = self.fetch_order(real_order_id, pair) - self._log_exchange_response('fetch_stoploss_order1', order_reg) - order_reg['id_stop'] = order_reg['id'] - order_reg['id'] = order_id - order_reg['type'] = 'stoploss' - order_reg['status_stop'] = 'triggered' - return order_reg - order['type'] = 'stoploss' - return order + return self._convert_stop_order(pair, order_id, order) except ccxt.BaseError: pass raise RetryableOrderError( From 9e9f9b21e5a833a3213499950d25e6c809ebfd59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 04:24:42 +0000 Subject: [PATCH 12/17] Bump stable-baselines3 from 2.0.0a9 to 2.0.0a10 Bumps [stable-baselines3](https://github.com/DLR-RM/stable-baselines3) from 2.0.0a9 to 2.0.0a10. - [Release notes](https://github.com/DLR-RM/stable-baselines3/releases) - [Commits](https://github.com/DLR-RM/stable-baselines3/commits) --- updated-dependencies: - dependency-name: stable-baselines3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-freqai-rl.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index 0e9df61f1..de48a1da4 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -5,7 +5,7 @@ torch==2.0.1 #until these branches will be released we can use this gymnasium==0.28.1 -stable_baselines3==2.0.0a9 +stable_baselines3==2.0.0a10 sb3_contrib>=2.0.0a9 # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.65.0 From f647fb342bc8ac2dc1ae33001cb8e5a39009a911 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Thu, 25 May 2023 16:35:06 +0200 Subject: [PATCH 13/17] Update base_tensorboard.py Remove incorrect warning message. --- freqtrade/freqai/tensorboard/base_tensorboard.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/tensorboard/base_tensorboard.py b/freqtrade/freqai/tensorboard/base_tensorboard.py index c2d47137e..72f47111c 100644 --- a/freqtrade/freqai/tensorboard/base_tensorboard.py +++ b/freqtrade/freqai/tensorboard/base_tensorboard.py @@ -10,8 +10,7 @@ logger = logging.getLogger(__name__) class BaseTensorboardLogger: def __init__(self, logdir: Path, activate: bool = True): - logger.warning("Tensorboard is not installed, no logs will be written." - "Ensure torch is installed, or use the torch/RL docker images") + pass def log_scalar(self, tag: str, scalar_value: Any, step: int): return @@ -23,8 +22,7 @@ class BaseTensorboardLogger: class BaseTensorBoardCallback(TrainingCallback): def __init__(self, logdir: Path, activate: bool = True): - logger.warning("Tensorboard is not installed, no logs will be written." - "Ensure torch is installed, or use the torch/RL docker images") + pass def after_iteration( self, model, epoch: int, evals_log: TrainingCallback.EvalsLog From bd266f654ef708dc321f767bd92772fcf9794a43 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 May 2023 08:19:50 +0200 Subject: [PATCH 14/17] Properly handle invalid pairlists type before config validation closes #8695 --- freqtrade/configuration/config_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 0ee48cf91..f1745df61 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -174,7 +174,7 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: return for pl in conf.get('pairlists', [{'method': 'StaticPairList'}]): - if (pl.get('method') == 'StaticPairList' + if (isinstance(pl, dict) and pl.get('method') == 'StaticPairList' and not conf.get('exchange', {}).get('pair_whitelist')): raise OperationalException("StaticPairList requires pair_whitelist to be set.") From df5e6409a46d06b782068cb684c4a1c9389a5b92 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 May 2023 20:18:39 +0200 Subject: [PATCH 15/17] Bump develop version to 2023.6-dev --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index f8818c35c..8f7717dd2 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.5.dev' +__version__ = '2023.6.dev' if 'dev' in __version__: from pathlib import Path From 8ec0469b111e4bbb3793975556adb9dbd1f25611 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 May 2023 09:53:46 +0200 Subject: [PATCH 16/17] Fix volatilityfilter behavior closes #8698 --- freqtrade/plugins/pairlist/VolatilityFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 9196026bb..baf4fcd26 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -74,7 +74,7 @@ class VolatilityFilter(IPairList): needed_pairs: ListPairsWithTimeframes = [ (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache] - since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days - 1)) + since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days)) # Get all candles candles = {} if needed_pairs: From 8a8616925685845afe2d4745d04f77786581edab Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 May 2023 09:59:57 +0200 Subject: [PATCH 17/17] Better handling of shift --- freqtrade/plugins/pairlist/VolatilityFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index baf4fcd26..61a1dcbf0 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -103,7 +103,7 @@ class VolatilityFilter(IPairList): result = False if daily_candles is not None and not daily_candles.empty: - returns = (np.log(daily_candles.close / daily_candles.close.shift(-1))) + returns = (np.log(daily_candles["close"].shift(1) / daily_candles["close"])) returns.fillna(0, inplace=True) volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days)