diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 9ed3d6dce..1487b92c2 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -85,6 +85,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `net_arch` | Network architecture which is well described in [`stable_baselines3` doc](https://stable-baselines3.readthedocs.io/en/master/guide/custom_policy.html#examples). In summary: `[, dict(vf=[], pi=[])]`. By default this is set to `[128, 128]`, which defines 2 shared hidden layers with 128 units each. | `randomize_starting_position` | Randomize the starting point of each episode to avoid overfitting.
**Datatype:** bool.
Default: `False`. | `drop_ohlc_from_features` | Do not include the normalized ohlc data in the feature set passed to the agent during training (ohlc will still be used for driving the environment in all cases)
**Datatype:** Boolean.
**Default:** `False` +| `progress_bar` | Display a progress bar with the current progress, elapsed time and estimated remaining time.
**Datatype:** Boolean.
Default: `False`. ### PyTorch parameters diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 62b0f83be..b8e240419 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -600,6 +600,7 @@ CONF_SCHEMA = { "policy_type": {"type": "string", "default": "MlpPolicy"}, "net_arch": {"type": "array", "default": [128, 128]}, "randomize_starting_position": {"type": "boolean", "default": False}, + "progress_bar": {"type": "boolean", "default": True}, "model_reward_parameters": { "type": "object", "properties": { diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index df10e40e5..8092d5af8 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -6,6 +6,7 @@ from freqtrade.exchange.exchange import Exchange from freqtrade.exchange.binance import Binance from freqtrade.exchange.bitpanda import Bitpanda from freqtrade.exchange.bittrex import Bittrex +from freqtrade.exchange.bitvavo import Bitvavo from freqtrade.exchange.bybit import Bybit from freqtrade.exchange.coinbasepro import Coinbasepro from freqtrade.exchange.exchange_utils import (ROUND_DOWN, ROUND_UP, amount_to_contract_precision, diff --git a/freqtrade/exchange/bitvavo.py b/freqtrade/exchange/bitvavo.py new file mode 100644 index 000000000..ba1d355cc --- /dev/null +++ b/freqtrade/exchange/bitvavo.py @@ -0,0 +1,23 @@ +"""Kucoin exchange subclass.""" +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Bitvavo(Exchange): + """Bitvavo exchange class. + + Contains adjustments needed for Freqtrade to work with this exchange. + + Please note that this exchange is not included in the list of exchanges + officially supported by the Freqtrade development team. So some features + may still not work as expected. + """ + + _ft_has: Dict = { + "ohlcv_candle_limit": 1440, + } diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ba7d79f97..7e276d538 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2371,12 +2371,12 @@ class Exchange: # Must fetch the leverage tiers for each market separately # * This is slow(~45s) on Okx, makes ~90 api calls to load all linear swap markets markets = self.markets - symbols = [] - for symbol, market in markets.items(): + symbols = [ + symbol for symbol, market in markets.items() if (self.market_is_future(market) - and market['quote'] == self._config['stake_currency']): - symbols.append(symbol) + and market['quote'] == self._config['stake_currency']) + ] tiers: Dict[str, List[Dict]] = {} @@ -2396,25 +2396,26 @@ class Exchange: else: logger.info("Using cached leverage_tiers.") - async def gather_results(): + async def gather_results(input_coro): return await asyncio.gather(*input_coro, return_exceptions=True) for input_coro in chunks(coros, 100): with self._loop_lock: - results = self.loop.run_until_complete(gather_results()) + results = self.loop.run_until_complete(gather_results(input_coro)) - for symbol, res in results: - tiers[symbol] = res + for res in results: + if isinstance(res, Exception): + logger.warning(f"Leverage tier exception: {repr(res)}") + continue + symbol, tier = res + tiers[symbol] = tier if len(coros) > 0: self.cache_leverage_tiers(tiers, self._config['stake_currency']) logger.info(f"Done initializing {len(symbols)} markets.") return tiers - else: - return {} - else: - return {} + return {} def cache_leverage_tiers(self, tiers: Dict[str, List[Dict]], stake_currency: str) -> None: @@ -2430,14 +2431,17 @@ class Exchange: def load_cached_leverage_tiers(self, stake_currency: str) -> Optional[Dict[str, List[Dict]]]: filename = self._config['datadir'] / "futures" / f"leverage_tiers_{stake_currency}.json" if filename.is_file(): - tiers = file_load_json(filename) - updated = tiers.get('updated') - if updated: - updated_dt = parser.parse(updated) - if updated_dt < datetime.now(timezone.utc) - timedelta(weeks=4): - logger.info("Cached leverage tiers are outdated. Will update.") - return None - return tiers['data'] + try: + tiers = file_load_json(filename) + updated = tiers.get('updated') + if updated: + updated_dt = parser.parse(updated) + if updated_dt < datetime.now(timezone.utc) - timedelta(weeks=4): + logger.info("Cached leverage tiers are outdated. Will update.") + return None + return tiers['data'] + except Exception: + logger.exception("Error loading cached leverage tiers. Refreshing.") return None def fill_leverage_tiers(self) -> None: diff --git a/freqtrade/freqai/prediction_models/ReinforcementLearner.py b/freqtrade/freqai/prediction_models/ReinforcementLearner.py index e795703d4..65990da87 100644 --- a/freqtrade/freqai/prediction_models/ReinforcementLearner.py +++ b/freqtrade/freqai/prediction_models/ReinforcementLearner.py @@ -71,7 +71,8 @@ class ReinforcementLearner(BaseReinforcementLearningModel): model.learn( total_timesteps=int(total_timesteps), - callback=[self.eval_callback, self.tensorboard_callback] + callback=[self.eval_callback, self.tensorboard_callback], + progress_bar=self.rl_config.get('progress_bar', False) ) if Path(dk.data_path / "best_model.zip").is_file(): diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index b4925770d..cbae6aede 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -273,7 +273,8 @@ def _get_resample_from_period(period: str) -> str: if period == 'day': return '1d' if period == 'week': - return '1w' + # Weekly defaulting to Monday. + return '1W-MON' if period == 'month': return '1M' raise ValueError(f"Period {period} is not supported.") diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 8ea70bb69..8aa706e62 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -303,11 +303,11 @@ def get_strategy(strategy: str, config=Depends(get_config)): @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) def list_freqaimodels(config=Depends(get_config)): from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver - strategies = FreqaiModelResolver.search_all_objects( + models = FreqaiModelResolver.search_all_objects( config, False) - strategies = sorted(strategies, key=lambda x: x['name']) + models = sorted(models, key=lambda x: x['name']) - return {'freqaimodels': [x['name'] for x in strategies]} + return {'freqaimodels': [x['name'] for x in models]} @router.get('/available_pairs', response_model=AvailablePairs, tags=['candle data']) diff --git a/pyproject.toml b/pyproject.toml index baf707c68..28de6a1d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ target-version = "py38" extend-select = [ "C90", # mccabe # "N", # pep8-naming + "F", # pyflakes + "E", # pycodestyle + "W", # pycodestyle "UP", # pyupgrade "TID", # flake8-tidy-imports # "EXE", # flake8-executable diff --git a/requirements-dev.txt b/requirements-dev.txt index 690a7ee70..fc0efcfe7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ coveralls==3.3.1 ruff==0.0.261 mypy==1.2.0 pre-commit==3.2.2 -pytest==7.3.0 +pytest==7.3.1 pytest-asyncio==0.21.0 pytest-cov==4.0.0 pytest-mock==3.10.0 @@ -19,7 +19,7 @@ isort==5.12.0 # For datetime mocking time-machine==2.9.0 # fastapi testing -httpx==0.23.3 +httpx==0.24.0 # Convert jupyter notebooks to markdown documents nbconvert==7.3.1 diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index 97d8e2c9b..45ccc40cc 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -5,8 +5,9 @@ torch==1.13.1; python_version < '3.11' #until these branches will be released we can use this gymnasium==0.28.1 -stable_baselines3>=2.0.0a1 -sb3_contrib>=2.0.0a1 +stable_baselines3==2.0.0a5 +sb3_contrib>=2.0.0a4 # Gym is forced to this version by stable-baselines3. setuptools==65.5.1 # Should be removed when gym is fixed. - +# Progress bar for stable-baselines3 and sb3-contrib +tqdm==4.65.0; python_version < '3.11' diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 840598d23..51396ab91 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -8,4 +8,4 @@ joblib==1.2.0 catboost==1.1.1; platform_machine != 'aarch64' and 'arm' not in platform_machine and python_version < '3.11' lightgbm==3.3.5 xgboost==1.7.5 -tensorboard==2.12.1 +tensorboard==2.12.2 diff --git a/requirements.txt b/requirements.txt index e09f8ba77..7db852961 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,8 @@ numpy==1.24.2 pandas==1.5.3 pandas-ta==0.3.14b -ccxt==3.0.59 -cryptography==40.0.1 +ccxt==3.0.69 +cryptography==40.0.2 aiohttp==3.8.4 SQLAlchemy==2.0.9 python-telegram-bot==13.15 @@ -20,7 +20,7 @@ jinja2==3.1.2 tables==3.8.0 blosc==1.11.1 joblib==1.2.0 -rich==13.3.3 +rich==13.3.4 pyarrow==11.0.0; platform_machine != 'armv7l' # find first, C search in arrays @@ -35,7 +35,7 @@ orjson==3.8.10 sdnotify==0.3.2 # API Server -fastapi==0.95.0 +fastapi==0.95.1 pydantic==1.10.7 uvicorn==0.21.1 pyjwt==2.6.0 diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 4a65b16d7..60855ca54 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -528,9 +528,11 @@ class TestCCXTExchange(): assert res[1] == timeframe assert res[2] == candle_type candles = res[3] - candle_count = exchange.ohlcv_candle_limit(timeframe, candle_type, since_ms) * 0.9 - candle_count1 = (now.timestamp() * 1000 - since_ms) // timeframe_ms - assert len(candles) >= min(candle_count, candle_count1) + factor = 0.9 + candle_count = exchange.ohlcv_candle_limit(timeframe, candle_type, since_ms) * factor + candle_count1 = (now.timestamp() * 1000 - since_ms) // timeframe_ms * factor + assert len(candles) >= min(candle_count, candle_count1), \ + f"{len(candles)} < {candle_count} in {timeframe}, Offset: {offset} {factor}" assert candles[0][0] == since_ms or (since_ms + timeframe_ms) def test_ccxt__async_get_candle_history(self, exchange: EXCHANGE_FIXTURE_TYPE): diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 7a3fa22f0..3824eddb7 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -1,14 +1,14 @@ from datetime import datetime, timedelta, timezone from pathlib import Path -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import AsyncMock, MagicMock, PropertyMock import ccxt import pytest from freqtrade.enums import CandleType, MarginMode, TradingMode -from freqtrade.exceptions import RetryableOrderError +from freqtrade.exceptions import RetryableOrderError, TemporaryError from freqtrade.exchange.exchange import timeframe_to_minutes -from tests.conftest import EXMS, get_mock_coro, get_patched_exchange, log_has +from tests.conftest import EXMS, get_patched_exchange, log_has from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -278,7 +278,7 @@ def test_load_leverage_tiers_okx(default_conf, mocker, markets, tmpdir, caplog, 'fetchLeverageTiers': False, 'fetchMarketLeverageTiers': True, }) - api_mock.fetch_market_leverage_tiers = get_mock_coro(side_effect=[ + api_mock.fetch_market_leverage_tiers = AsyncMock(side_effect=[ [ { 'tier': 1, @@ -341,6 +341,7 @@ def test_load_leverage_tiers_okx(default_conf, mocker, markets, tmpdir, caplog, } }, ], + TemporaryError("this Failed"), [ { 'tier': 1, diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 0cc32baaf..9fdd0d61e 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -465,7 +465,7 @@ def test_generate_periodic_breakdown_stats(testdatadir): def test__get_resample_from_period(): assert _get_resample_from_period('day') == '1d' - assert _get_resample_from_period('week') == '1w' + assert _get_resample_from_period('week') == '1W-MON' assert _get_resample_from_period('month') == '1M' with pytest.raises(ValueError, match=r"Period noooo is not supported."): _get_resample_from_period('noooo')