diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b077be04..bb5bc209e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -351,7 +351,7 @@ jobs: python setup.py sdist bdist_wheel - name: Publish to PyPI (Test) - uses: pypa/gh-action-pypi-publish@v1.5.0 + uses: pypa/gh-action-pypi-publish@v1.5.1 if: (github.event_name == 'release') with: user: __token__ @@ -359,7 +359,7 @@ jobs: repository_url: https://test.pypi.org/legacy/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.5.0 + uses: pypa/gh-action-pypi-publish@v1.5.1 if: (github.event_name == 'release') with: user: __token__ diff --git a/.gitignore b/.gitignore index d6cec5225..e400c01f5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,10 +7,15 @@ logfile.txt user_data/* !user_data/strategy/sample_strategy.py !user_data/notebooks +!user_data/models +!user_data/freqaimodels +user_data/freqaimodels/* +user_data/models/* user_data/notebooks/* freqtrade-plot.html freqtrade-profit-plot.html freqtrade/rpc/api_server/ui/* +build_helpers/ta-lib/* # Macos related .DS_Store @@ -107,3 +112,4 @@ target/ !config_examples/config_ftx.example.json !config_examples/config_full.example.json !config_examples/config_kraken.example.json +!config_examples/config_freqai.example.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 759ac0a6a..86c4ec1ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.3 + - types-requests==2.28.9 - types-tabulate==0.8.11 - types-python-dateutil==2.8.19 # stages: [push] diff --git a/Dockerfile b/Dockerfile index 5138ecec9..e84a4d095 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.5-slim-bullseye as base +FROM python:3.10.6-slim-bullseye as base # Setup env ENV LANG C.UTF-8 @@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker" # Prepare environment RUN mkdir /freqtrade \ && apt-get update \ - && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev libgomp1 \ && apt-get clean \ && useradd -u 1000 -G sudo -U -m -s /bin/bash ftuser \ && chown ftuser:ftuser /freqtrade \ diff --git a/README.md b/README.md index 881895c9a..0cc2364e5 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ Please find the complete documentation on the [freqtrade website](https://www.fr - [x] **Dry-run**: Run the bot without paying money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. - [x] **Strategy Optimization by machine learning**: Use machine learning to optimize your buy/sell strategy parameters with real exchange data. +- [X] **Adaptive prediction modeling**: Build a smart strategy with FreqAI that self-trains to the market via adaptive machine learning methods. [Learn more](https://www.freqtrade.io/en/stable/freqai/) - [x] **Edge position sizing** Calculate your win rate, risk reward ratio, the best stoploss and adjust your position size before taking a position for each specific market. [Learn more](https://www.freqtrade.io/en/stable/edge/). - [x] **Whitelist crypto-currencies**: Select which crypto-currency you want to trade or use dynamic whitelists. - [x] **Blacklist crypto-currencies**: Select which crypto-currency you want to avoid. @@ -129,7 +130,7 @@ Telegram is not mandatory. However, this is a great way to control your bot. Mor - `/start`: Starts the trader. - `/stop`: Stops the trader. -- `/stopbuy`: Stop entering new trades. +- `/stopentry`: Stop entering new trades. - `/status |[table]`: Lists all or specific open trades. - `/profit []`: Lists cumulative profit from all finished trades, over the last n days. - `/forceexit |all`: Instantly exits the given trade (Ignoring `minimum_roi`). @@ -193,7 +194,7 @@ Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/ The clock must be accurate, synchronized to a NTP server very frequently to avoid problems with communication to the exchanges. -### Min hardware required +### Minimum hardware required To run this bot we recommend you a cloud instance with a minimum of: diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index 00c4417ae..079d578b4 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -4,7 +4,7 @@ else INSTALL_LOC=${1} fi echo "Installing to ${INSTALL_LOC}" -if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then +if [ -n "$2" ] || [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz cd ta-lib \ && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ @@ -17,11 +17,17 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then cd .. && rm -rf ./ta-lib/ exit 1 fi - which sudo && sudo make install || make install - if [ -x "$(command -v apt-get)" ]; then - echo "Updating library path using ldconfig" - sudo ldconfig + if [ -z "$2" ]; then + which sudo && sudo make install || make install + if [ -x "$(command -v apt-get)" ]; then + echo "Updating library path using ldconfig" + sudo ldconfig + fi + else + # Don't install with sudo + make install fi + cd .. && rm -rf ./ta-lib/ else echo "TA-lib already installed, skipping installation" diff --git a/build_helpers/publish_docker_arm64.sh b/build_helpers/publish_docker_arm64.sh index 70f99e54b..4c66f4483 100755 --- a/build_helpers/publish_docker_arm64.sh +++ b/build_helpers/publish_docker_arm64.sh @@ -6,10 +6,12 @@ export DOCKER_BUILDKIT=1 # Replace / with _ to create a valid tag TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") TAG_PLOT=${TAG}_plot +TAG_FREQAI=${TAG}_freqai TAG_PI="${TAG}_pi" TAG_ARM=${TAG}_arm TAG_PLOT_ARM=${TAG_PLOT}_arm +TAG_FREQAI_ARM=${TAG_FREQAI}_arm CACHE_IMAGE=freqtradeorg/freqtrade_cache echo "Running for ${TAG}" @@ -38,8 +40,10 @@ fi docker tag freqtrade:$TAG_ARM ${CACHE_IMAGE}:$TAG_ARM docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_PLOT_ARM} -f docker/Dockerfile.plot . +docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_FREQAI_ARM} -f docker/Dockerfile.freqai . docker tag freqtrade:$TAG_PLOT_ARM ${CACHE_IMAGE}:$TAG_PLOT_ARM +docker tag freqtrade:$TAG_FREQAI_ARM ${CACHE_IMAGE}:$TAG_FREQAI_ARM # Run backtest docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 @@ -53,6 +57,7 @@ docker images # docker push ${IMAGE_NAME} docker push ${CACHE_IMAGE}:$TAG_PLOT_ARM +docker push ${CACHE_IMAGE}:$TAG_FREQAI_ARM docker push ${CACHE_IMAGE}:$TAG_ARM # Create multi-arch image @@ -66,6 +71,9 @@ docker manifest push -p ${IMAGE_NAME}:${TAG} docker manifest create ${IMAGE_NAME}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} ${CACHE_IMAGE}:${TAG_PLOT} docker manifest push -p ${IMAGE_NAME}:${TAG_PLOT} +docker manifest create ${IMAGE_NAME}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI_ARM} ${CACHE_IMAGE}:${TAG_FREQAI} +docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI} + # Tag as latest for develop builds if [ "${TAG}" = "develop" ]; then docker manifest create ${IMAGE_NAME}:latest ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG} diff --git a/build_helpers/publish_docker_multi.sh b/build_helpers/publish_docker_multi.sh index fd5f0ef93..c13732003 100755 --- a/build_helpers/publish_docker_multi.sh +++ b/build_helpers/publish_docker_multi.sh @@ -5,6 +5,7 @@ # Replace / with _ to create a valid tag TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") TAG_PLOT=${TAG}_plot +TAG_FREQAI=${TAG}_freqai TAG_PI="${TAG}_pi" PI_PLATFORM="linux/arm/v7" @@ -49,8 +50,10 @@ fi docker tag freqtrade:$TAG ${CACHE_IMAGE}:$TAG docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot . +docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG} -t freqtrade:${TAG_FREQAI} -f docker/Dockerfile.freqai . docker tag freqtrade:$TAG_PLOT ${CACHE_IMAGE}:$TAG_PLOT +docker tag freqtrade:$TAG_FREQAI ${CACHE_IMAGE}:$TAG_FREQAI # Run backtest docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 @@ -64,6 +67,7 @@ docker images docker push ${CACHE_IMAGE} docker push ${CACHE_IMAGE}:$TAG_PLOT +docker push ${CACHE_IMAGE}:$TAG_FREQAI docker push ${CACHE_IMAGE}:$TAG diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json new file mode 100644 index 000000000..aeb1cb13d --- /dev/null +++ b/config_examples/config_freqai.example.json @@ -0,0 +1,96 @@ +{ + "trading_mode": "futures", + "margin_mode": "isolated", + "max_open_trades": 5, + "stake_currency": "USDT", + "stake_amount": 200, + "tradable_balance_ratio": 1, + "fiat_display_currency": "USD", + "dry_run": true, + "timeframe": "3m", + "dry_run_wallet": 1000, + "cancel_open_orders_on_exit": true, + "unfilledtimeout": { + "entry": 10, + "exit": 30 + }, + "exchange": { + "name": "binance", + "key": "", + "secret": "", + "ccxt_config": { + "enableRateLimit": true + }, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 200 + }, + "pair_whitelist": [ + "1INCH/USDT", + "ALGO/USDT" + ], + "pair_blacklist": [] + }, + "entry_pricing": { + "price_side": "same", + "use_order_book": true, + "order_book_top": 1, + "price_last_balance": 0.0, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "exit_pricing": { + "price_side": "other", + "use_order_book": true, + "order_book_top": 1 + }, + "pairlists": [ + { + "method": "StaticPairList" + } + ], + "freqai": { + "enabled": true, + "startup_candles": 10000, + "purge_old_models": true, + "train_period_days": 15, + "backtest_period_days": 7, + "live_retrain_hours": 0, + "identifier": "uniqe-id", + "feature_parameters": { + "include_timeframes": [ + "3m", + "15m", + "1h" + ], + "include_corr_pairlist": [ + "BTC/USDT", + "ETH/USDT" + ], + "label_period_candles": 20, + "include_shifted_candles": 2, + "DI_threshold": 0.9, + "weight_factor": 0.9, + "principal_component_analysis": false, + "use_SVM_to_remove_outliers": true, + "stratify_training_data": 0, + "indicator_max_period_candles": 20, + "indicator_periods_candles": [10, 20] + }, + "data_split_parameters": { + "test_size": 0.33, + "random_state": 1 + }, + "model_training_parameters": { + "n_estimators": 1000 + } + }, + "bot_name": "", + "force_entry_enable": true, + "initial_state": "running", + "internals": { + "process_throttle_secs": 5 + } +} diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index e2e9a16fd..74457d2b6 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -5,6 +5,7 @@ "tradable_balance_ratio": 0.99, "fiat_display_currency": "USD", "amount_reserve_percent": 0.05, + "available_capital": 1000, "amend_last_stake_amount": false, "last_stake_amount_min_ratio": 0.5, "dry_run": true, @@ -92,6 +93,7 @@ "secret": "your_exchange_secret", "password": "", "log_responses": false, + // "unknown_fee_rate": 1, "ccxt_config": {}, "ccxt_async_config": {}, "pair_whitelist": [ diff --git a/docker/Dockerfile.freqai b/docker/Dockerfile.freqai new file mode 100644 index 000000000..9a2f75700 --- /dev/null +++ b/docker/Dockerfile.freqai @@ -0,0 +1,9 @@ +ARG sourceimage=freqtradeorg/freqtrade +ARG sourcetag=develop +FROM ${sourceimage}:${sourcetag} + +# Install dependencies +COPY requirements-freqai.txt /freqtrade/ + +RUN pip install -r requirements-freqai.txt --user --no-cache-dir + diff --git a/docs/assets/freqai_DI.jpg b/docs/assets/freqai_DI.jpg new file mode 100644 index 000000000..5e2aead34 Binary files /dev/null and b/docs/assets/freqai_DI.jpg differ diff --git a/docs/assets/freqai_algo.jpg b/docs/assets/freqai_algo.jpg new file mode 100644 index 000000000..44600e71a Binary files /dev/null and b/docs/assets/freqai_algo.jpg differ diff --git a/docs/assets/freqai_dbscan.jpg b/docs/assets/freqai_dbscan.jpg new file mode 100644 index 000000000..0974550d2 Binary files /dev/null and b/docs/assets/freqai_dbscan.jpg differ diff --git a/docs/assets/freqai_doc_logo.svg b/docs/assets/freqai_doc_logo.svg new file mode 100644 index 000000000..23b116883 --- /dev/null +++ b/docs/assets/freqai_doc_logo.svg @@ -0,0 +1,304 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FreqAI + + + + + + diff --git a/docs/assets/freqai_moving-window.jpg b/docs/assets/freqai_moving-window.jpg new file mode 100644 index 000000000..9361479ac Binary files /dev/null and b/docs/assets/freqai_moving-window.jpg differ diff --git a/docs/assets/freqai_weight-factor.jpg b/docs/assets/freqai_weight-factor.jpg new file mode 100644 index 000000000..4f8b23e18 Binary files /dev/null and b/docs/assets/freqai_weight-factor.jpg differ diff --git a/docs/backtesting.md b/docs/backtesting.md index 50fc96923..8b2fdc345 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -514,6 +514,7 @@ You can then load the trades to perform further analysis as shown in the [data a Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions: +- Exchange [trading limits](#trading-limits-in-backtesting) are respected - Buys happen at open-price - All orders are filled at the requested price (no slippage, no unfilled orders) - Exit-signal exits happen at open-price of the consecutive candle @@ -543,7 +544,32 @@ Also, keep in mind that past results don't guarantee future success. In addition to the above assumptions, strategy authors should carefully read the [Common Mistakes](strategy-customization.md#common-mistakes-when-developing-strategies) section, to avoid using data in backtesting which is not available in real market conditions. -### Improved backtest accuracy +### Trading limits in backtesting + +Exchanges have certain trading limits, like minimum base currency, or minimum stake (quote) currency. +These limits are usually listed in the exchange documentation as "trading rules" or similar. + +Backtesting (as well as live and dry-run) does honor these limits, and will ensure that a stoploss can be placed below this value - so the value will be slightly higher than what the exchange specifies. +Freqtrade has however no information about historic limits. + +This can lead to situations where trading-limits are inflated by using a historic price, resulting in minimum amounts > 50$. + +For example: + +BTC minimum tradable amount is 0.001. +BTC trades at 22.000\$ today (0.001 BTC is related to this) - but the backtesting period includes prices as high as 50.000\$. +Today's minimum would be `0.001 * 22_000` - or 22\$. +However the limit could also be 50$ - based on `0.001 * 50_000` in some historic setting. + +#### Trading precision limits + +Most exchanges pose precision limits on both price and amounts, so you cannot buy 1.0020401 of a pair, or at a price of 1.24567123123. +Instead, these prices and amounts will be rounded or truncated (based on the exchange definition) to the defined trading precision. +The above values may for example be rounded to an amount of 1.002, and a price of 1.24567. + +These precision values are based on current exchange limits (as described in the [above section](#trading-limits-in-backtesting)), as historic precision limits are not available. + +## Improved backtest accuracy One big limitation of backtesting is it's inability to know how prices moved intra-candle (was high before close, or viceversa?). So assuming you run backtesting with a 1h timeframe, there will be 4 prices for that candle (Open, High, Low, Close). diff --git a/docs/bot-basics.md b/docs/bot-basics.md index 14823722e..3df926371 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -70,7 +70,7 @@ This loop will be repeated again and again until the bot is stopped. * Determine stake size by calling the `custom_stake_amount()` callback. * Check position adjustments for open trades if enabled and call `adjust_trade_position()` to determine if an additional order is requested. * Call `custom_stoploss()` and `custom_exit()` to find custom exit points. - * For exits based on exit-signal and custom-exit: Call `custom_exit_price()` to determine exit price (Prices are moved to be within the closing candle). + * For exits based on exit-signal, custom-exit and partial exits: Call `custom_exit_price()` to determine exit price (Prices are moved to be within the closing candle). * Generate backtest report output !!! Note diff --git a/docs/configuration.md b/docs/configuration.md index 412571674..d5c0b3d8b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -105,7 +105,7 @@ This is similar to using multiple `--config` parameters, but simpler in usage as ``` json title="Result" { - "max_open_trades": 10, + "max_open_trades": 3, "stake_currency": "USDT", "stake_amount": "unlimited" } diff --git a/docs/data-download.md b/docs/data-download.md index 681fb717d..b72e7f337 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -63,7 +63,7 @@ optional arguments: `jsongz`). --trading-mode {spot,margin,futures} Select Trading mode - --prepend Allow data prepending. + --prepend Allow data prepending. (Data-appending is disabled) Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -186,7 +186,7 @@ Freqtrade currently supports 3 data-formats for both OHLCV and trades data: By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively. -To persist this change, you can should also add the following snippet to your configuration, so you don't have to insert the above arguments each time: +To persist this change, you should also add the following snippet to your configuration, so you don't have to insert the above arguments each time: ``` jsonc // ... @@ -374,6 +374,7 @@ usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--data-format-ohlcv {json,jsongz,hdf5}] [-p PAIRS [PAIRS ...]] [--trading-mode {spot,margin,futures}] + [--show-timerange] optional arguments: -h, --help show this help message and exit @@ -387,6 +388,8 @@ optional arguments: separated. --trading-mode {spot,margin,futures} Select Trading mode + --show-timerange Show timerange available for available data. (May take + a while to calculate). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/developer.md b/docs/developer.md index 0209d220a..aca4ce4ed 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -68,6 +68,36 @@ def test_method_to_test(caplog): ``` +### Debug configuration + +To debug freqtrade, we recommend VSCode with the following launch configuration (located in `.vscode/launch.json`). +Details will obviously vary between setups - but this should work to get you started. + +``` json +{ + "name": "freqtrade trade", + "type": "python", + "request": "launch", + "module": "freqtrade", + "console": "integratedTerminal", + "args": [ + "trade", + // Optional: + // "--userdir", "user_data", + "--strategy", + "MyAwesomeStrategy", + ] +}, +``` + +Command line arguments can be added in the `"args"` array. +This method can also be used to debug a strategy, by setting the breakpoints within the strategy. + +A similar setup can also be taken for Pycharm - using `freqtrade` as module name, and setting the command line arguments as "parameters". + +!!! Note "Startup directory" + This assumes that you have the repository checked out, and the editor is started at the repository root level (so setup.py is at the top level of your repository). + ## ErrorHandling Freqtrade Exceptions all inherit from `FreqtradeException`. diff --git a/docs/faq.md b/docs/faq.md index f1542d08e..381bbceb5 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -77,9 +77,9 @@ Freqtrade will not provide incomplete candles to strategies. Using incomplete ca You can use "current" market data by using the [dataprovider](strategy-customization.md#orderbookpair-maximum)'s orderbook or ticker methods - which however cannot be used during backtesting. -### Is there a setting to only SELL the coins being held and not perform anymore BUYS? +### Is there a setting to only Exit the trades being held and not perform any new Entries? -You can use the `/stopbuy` command in Telegram to prevent future buys, followed by `/forceexit all` (sell all open trades). +You can use the `/stopentry` command in Telegram to prevent future trade entry, followed by `/forceexit all` (sell all open trades). ### I want to run multiple bots on the same machine diff --git a/docs/freqai.md b/docs/freqai.md new file mode 100644 index 000000000..3d10280dd --- /dev/null +++ b/docs/freqai.md @@ -0,0 +1,759 @@ +![freqai-logo](assets/freqai_doc_logo.svg) + +# FreqAI + +FreqAI is a module designed to automate a variety of tasks associated with training a predictive machine learning model to generate market forecasts given a set of input features. + +Features include: + +* **Self-adaptive retraining**: retrain models during [live deployments](#running-the-model-live) to self-adapt to the market in an unsupervised manner. +* **Rapid feature engineering**: create large rich [feature sets](#feature-engineering) (10k+ features) based on simple user-created strategies. +* **High performance**: adaptive retraining occurs on a separate thread (or on GPU if available) from inferencing and bot trade operations. Newest models and data are kept in memory for rapid inferencing. +* **Realistic backtesting**: emulate self-adaptive retraining with a [backtesting module](#backtesting) that automates past retraining. +* **Modifiability**: use the generalized and robust architecture for incorporating any [machine learning library/method](#building-a-custom-prediction-model) available in Python. Eight examples are currently available, including classifiers, regressors, and a convolutional neural network. +* **Smart outlier removal**: remove outliers from training and prediction data sets using a variety of [outlier detection techniques](#outlier-removal). +* **Crash resilience**: store model to disk to make reloading from a crash fast and easy, and [purge obsolete files](#purging-old-model-data) for sustained dry/live runs. +* **Automatic data normalization**: [normalize the data](#feature-normalization) in a smart and statistically safe way. +* **Automatic data download**: compute the data download timerange and update historic data (in live deployments). +* **Cleaning of incoming data**: handle NaNs safely before training and prediction. +* **Dimensionality reduction**: reduce the size of the training data via [Principal Component Analysis](#reducing-data-dimensionality-with-principal-component-analysis). +* **Deploying bot fleets**: set one bot to train models while a fleet of [follower bots](#setting-up-a-follower) inference the models and handle trades. + +## Quick start + +The easiest way to quickly test FreqAI is to run it in dry mode with the following command + +```bash +freqtrade trade --config config_examples/config_freqai.example.json --strategy FreqaiExampleStrategy --freqaimodel LightGBMRegressor --strategy-path freqtrade/templates +``` + +The user will see the boot-up process of automatic data downloading, followed by simultaneous training and trading. + +The example strategy, example prediction model, and example config can be found in +`freqtrade/templates/FreqaiExampleStrategy.py`, `freqtrade/freqai/prediction_models/LightGBMRegressor.py`, and +`config_examples/config_freqai.example.json`, respectively. + +## General approach + +The user provides FreqAI with a set of custom *base* indicators (the same way as in a typical Freqtrade strategy) as well as target values (*labels*). +FreqAI trains a model to predict the target values based on the input of custom indicators, for each pair in the whitelist. These models are consistently retrained to adapt to market conditions. FreqAI offers the ability to both backtest strategies (emulating reality with periodic retraining) and deploy dry/live runs. In dry/live conditions, FreqAI can be set to constant retraining in a background thread in an effort to keep models as up to date as possible. + +An overview of the algorithm is shown below, explaining the data processing pipeline and the model usage. + +![freqai-algo](assets/freqai_algo.jpg) + +### Important machine learning vocabulary + +**Features** - the quantities with which a model is trained. All features for a single candle is stored as a vector. In FreqAI, the user +builds the feature sets from anything they can construct in the strategy. + +**Labels** - the target values that a model is trained +toward. Each set of features is associated with a single label that is +defined by the user within the strategy. These labels intentionally look into the +future, and are not available to the model during dry/live/backtesting. + +**Training** - the process of feeding individual feature sets, composed of historic data, with associated labels into the +model with the goal of matching input feature sets to associated labels. + +**Train data** - a subset of the historic data that is fed to the model during +training. This data directly influences weight connections in the model. + +**Test data** - a subset of the historic data that is used to evaluate the performance of the model after training. This data does not influence nodal weights within the model. + +## Install prerequisites + +The normal Freqtrade install process will ask the user if they wish to install FreqAI dependencies. The user should reply "yes" to this question if they wish to use FreqAI. If the user did not reply yes, they can manually install these dependencies after the install with: + +``` bash +pip install -r requirements-freqai.txt +``` + +!!! Note + Catboost will not be installed on arm devices (raspberry, Mac M1, ARM based VPS, ...), since Catboost does not provide wheels for this platform. + +### Usage with docker + +For docker users, a dedicated tag with freqAI dependencies is available as `:freqai`. +As such - you can replace the image line in your docker-compose file with `image: freqtradeorg/freqtrade:develop_freqai`. +This image contains the regular freqAI dependencies. Similar to native installs, Catboost will not be available on ARM based devices. + +## Setting up FreqAI + +### Parameter table + +The table below will list all configuration parameters available for FreqAI, presented in the same order as `config_examples/config_freqai.example.json`. + +Mandatory parameters are marked as **Required**, which means that they are required to be set in one of the possible ways. + +| Parameter | Description | +|------------|-------------| +| | **General configuration parameters** +| `freqai` | **Required.**
The parent dictionary containing all the parameters for controlling FreqAI.
**Datatype:** Dictionary. +| `startup_candles` | Number of candles needed for *backtesting only* to ensure all indicators are non NaNs at the start of the first train period.
**Datatype:** Positive integer. +| `purge_old_models` | Delete obsolete models (otherwise, all historic models will remain on disk).
**Datatype:** Boolean. Default: `False`. +| `train_period_days` | **Required.**
Number of days to use for the training data (width of the sliding window).
**Datatype:** Positive integer. +| `backtest_period_days` | **Required.**
Number of days to inference from the trained model before sliding the window defined above, and retraining the model. This can be fractional days, but beware that the user-provided `timerange` will be divided by this number to yield the number of trainings necessary to complete the backtest.
**Datatype:** Float. +| `identifier` | **Required.**
A unique name for the current model. This can be reused to reload pre-trained models/data.
**Datatype:** String. +| `live_retrain_hours` | Frequency of retraining during dry/live runs.
Default set to 0, which means the model will retrain as often as possible.
**Datatype:** Float > 0. +| `expiration_hours` | Avoid making predictions if a model is more than `expiration_hours` old.
Defaults set to 0, which means models never expire.
**Datatype:** Positive integer. +| `fit_live_predictions_candles` | Number of historical candles to use for computing target (label) statistics from prediction data, instead of from the training data set.
**Datatype:** Positive integer. +| `follow_mode` | If true, this instance of FreqAI will look for models associated with `identifier` and load those for inferencing. A `follower` will **not** train new models.
**Datatype:** Boolean. Default: `False`. +| | **Feature parameters** +| `feature_parameters` | A dictionary containing the parameters used to engineer the feature set. Details and examples are shown [here](#feature-engineering).
**Datatype:** Dictionary. +| `include_timeframes` | A list of timeframes that all indicators in `populate_any_indicators` will be created for. The list is added as features to the base asset feature set.
**Datatype:** List of timeframes (strings). +| `include_corr_pairlist` | A list of correlated coins that FreqAI will add as additional features to all `pair_whitelist` coins. All indicators set in `populate_any_indicators` during feature engineering (see details [here](#feature-engineering)) will be created for each coin in this list, and that set of features is added to the base asset feature set.
**Datatype:** List of assets (strings). +| `label_period_candles` | Number of candles into the future that the labels are created for. This is used in `populate_any_indicators` (see `templates/FreqaiExampleStrategy.py` for detailed usage). The user can create custom labels, making use of this parameter or not.
**Datatype:** Positive integer. +| `include_shifted_candles` | Add features from previous candles to subsequent candles to add historical information. FreqAI takes all features from the `include_shifted_candles` previous candles, duplicates and shifts them so that the information is available for the subsequent candle.
**Datatype:** Positive integer. +| `weight_factor` | Used to set weights for training data points according to their recency. See details about how it works [here](#controlling-the-model-learning-process).
**Datatype:** Positive float (typically < 1). +| `indicator_max_period_candles` | The maximum period used in `populate_any_indicators()` for indicator creation. FreqAI uses this information in combination with the maximum timeframe to calculate how many data points that should be downloaded so that the first data point does not have a NaN.
**Datatype:** Positive integer. +| `indicator_periods_candles` | Calculate indicators for `indicator_periods_candles` time periods and add them to the feature set.
**Datatype:** List of positive integers. +| `stratify_training_data` | This value is used to indicate the grouping of the data. For example, 2 would set every 2nd data point into a separate dataset to be pulled from during training/testing. See details about how it works [here](#stratifying-the-data-for-training-and-testing-the-model)
**Datatype:** Positive integer. +| `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](#reducing-data-dimensionality-with-principal-component-analysis)
**Datatype:** Boolean. +| `DI_threshold` | Activates the Dissimilarity Index for outlier detection when > 0. See details about how it works [here](#removing-outliers-with-the-dissimilarity-index).
**Datatype:** Positive float (typically < 1). +| `use_SVM_to_remove_outliers` | Train a support vector machine to detect and remove outliers from the training data set, as well as from incoming data points. See details about how it works [here](#removing-outliers-using-a-support-vector-machine-svm).
**Datatype:** Boolean. +| `svm_params` | All parameters available in Sklearn's `SGDOneClassSVM()`. See details about some select parameters [here](#removing-outliers-using-a-support-vector-machine-svm).
**Datatype:** Dictionary. +| `use_DBSCAN_to_remove_outliers` | Cluster data using DBSCAN to identify and remove outliers from training and prediction data. See details about how it works [here](#removing-outliers-with-dbscan).
**Datatype:** Boolean. +| `outlier_protection_percentage` | If more than `outlier_protection_percentage` fraction of points are removed as outliers, FreqAI will log a warning message and ignore outlier detection while keeping the original dataset intact.
**Datatype:** float. Default: `30` +| | **Data split parameters** +| `data_split_parameters` | Include any additional parameters available from Scikit-learn `test_train_split()`, which are shown [here](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html) (external website).
**Datatype:** Dictionary. +| `test_size` | Fraction of data that should be used for testing instead of training.
**Datatype:** Positive float < 1. +| `shuffle` | Shuffle the training data points during training. Typically, for time-series forecasting, this is set to `False`.
+| | **Model training parameters** +| `model_training_parameters` | A flexible dictionary that includes all parameters available by the user selected model library. For example, if the user uses `LightGBMRegressor`, this dictionary can contain any parameter available by the `LightGBMRegressor` [here](https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMRegressor.html) (external website). If the user selects a different model, this dictionary can contain any parameter from that model.
**Datatype:** Dictionary.**Datatype:** Boolean. +| `n_estimators` | The number of boosted trees to fit in regression.
**Datatype:** Integer. +| `learning_rate` | Boosting learning rate during regression.
**Datatype:** Float. +| `n_jobs`, `thread_count`, `task_type` | Set the number of threads for parallel processing and the `task_type` (`gpu` or `cpu`). Different model libraries use different parameter names.
**Datatype:** Float. +| | **Extraneous parameters** +| `keras` | If your model makes use of Keras (typical for Tensorflow-based prediction models), activate this flag so that the model save/loading follows Keras standards.
**Datatype:** Boolean. Default: `False`. +| `conv_width` | The width of a convolutional neural network input tensor. This replaces the need for shifting candles (`include_shifted_candles`) by feeding in historical data points as the second dimension of the tensor. Technically, this parameter can also be used for regressors, but it only adds computational overhead and does not change the model training/prediction.
**Datatype:** Integer. Default: 2. + +### Important dataframe key patterns + +Below are the values the user can expect to include/use inside a typical strategy dataframe (`df[]`): + +| DataFrame Key | Description | +|------------|-------------| +| `df['&*']` | Any dataframe column prepended with `&` in `populate_any_indicators()` is treated as a training target (label) inside FreqAI (typically following the naming convention `&-s*`). The names of these dataframe columns are fed back to the user as the predictions. For example, if the user wishes to predict the price change in the next 40 candles (similar to `templates/FreqaiExampleStrategy.py`), they set `df['&-s_close']`. FreqAI makes the predictions and gives them back under the same key (`df['&-s_close']`) to be used in `populate_entry/exit_trend()`.
**Datatype:** Depends on the output of the model. +| `df['&*_std/mean']` | Standard deviation and mean values of the user-defined labels during training (or live tracking with `fit_live_predictions_candles`). Commonly used to understand the rarity of a prediction (use the z-score as shown in `templates/FreqaiExampleStrategy.py` to evaluate how often a particular prediction was observed during training or historically with `fit_live_predictions_candles`).
**Datatype:** Float. +| `df['do_predict']` | Indication of an outlier data point. The return value is integer between -1 and 2, which lets the user know if the prediction is trustworthy or not. `do_predict==1` means the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](#removing-outliers-with-the-dissimilarity-index)) of the input data point is above the user-defined threshold, FreqAI will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers()` is active, the Support Vector Machine (SVM) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`.
**Datatype:** Integer between -1 and 2. +| `df['DI_values']` | Dissimilarity Index values are proxies to the level of confidence FreqAI has in the prediction. A lower DI means the prediction is close to the training data, i.e., higher prediction confidence.
**Datatype:** Float. +| `df['%*']` | Any dataframe column prepended with `%` in `populate_any_indicators()` is treated as a training feature. For example, the user can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](#feature-engineering).
**Note**: Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features is easily engineered using the multiplictative functionality described in the `feature_parameters` table shown above), these features are removed from the dataframe upon return from FreqAI. If the user wishes to keep a particular type of feature for plotting purposes, they can prepend it with `%%`.
**Datatype:** Depends on the output of the model. + +### File structure + +`user_data_dir/models/` contains all the data associated with the trainings and backtests. +This file structure is heavily controlled and inferenced by the `FreqaiDataKitchen()` +and should therefore not be modified. + +### Example config file + +The user interface is isolated to the typical Freqtrade config file. A FreqAI config should include: + +```json + "freqai": { + "enabled": true, + "startup_candles": 10000, + "purge_old_models": true, + "train_period_days": 30, + "backtest_period_days": 7, + "identifier" : "unique-id", + "feature_parameters" : { + "include_timeframes": ["5m","15m","4h"], + "include_corr_pairlist": [ + "ETH/USD", + "LINK/USD", + "BNB/USD" + ], + "label_period_candles": 24, + "include_shifted_candles": 2, + "indicator_max_period_candles": 20, + "indicator_periods_candles": [10, 20] + }, + "data_split_parameters" : { + "test_size": 0.25 + }, + "model_training_parameters" : { + "n_estimators": 100 + }, + } +``` + +## Building a FreqAI strategy + +The FreqAI strategy requires the user to include the following lines of code in the standard Freqtrade strategy: + +```python + + def informative_pairs(self): + whitelist_pairs = self.dp.current_whitelist() + corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] + informative_pairs = [] + for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: + for pair in whitelist_pairs: + informative_pairs.append((pair, tf)) + for pair in corr_pairs: + if pair in whitelist_pairs: + continue # avoid duplication + informative_pairs.append((pair, tf)) + return informative_pairs + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + # the model will return all labels created by user in `populate_any_indicators` + # (& appended targets), an indication of whether or not the prediction should be accepted, + # the target mean/std values for each of the labels created by user in + # `populate_any_indicators()` for each training period. + + dataframe = self.freqai.start(dataframe, metadata, self) + + return dataframe + + def populate_any_indicators( + self, pair, df, tf, informative=None, set_generalized_indicators=False + ): + """ + Function designed to automatically generate, name and merge features + from user indicated timeframes in the configuration file. User controls the indicators + passed to the training/prediction by prepending indicators with `'%-' + coin ` + (see convention below). I.e. user should not prepend any supporting metrics + (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the + model. + :param pair: pair to be used as informative + :param df: strategy dataframe which will receive merges from informatives + :param tf: timeframe of the dataframe which will modify the feature names + :param informative: the dataframe associated with the informative pair + :param coin: the name of the coin which will modify the feature names. + """ + + coin = pair.split('/')[0] + + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) + + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) + + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) + + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) + + return df + + +``` + +Notice how the `populate_any_indicators()` is where the user adds their own features ([more information](#feature-engineering)) and labels ([more information](#setting-classifier-targets)). See a full example at `templates/FreqaiExampleStrategy.py`. + +## Creating a dynamic target + +The `&*_std/mean` return values describe the statistical fit of the user defined label *during the most recent training*. This value allows the user to know the rarity of a given prediction. For example, `templates/FreqaiExampleStrategy.py`, creates a `target_roi` which is based on filtering out predictions that are below a given z-score of 1.25. + +```python +dataframe["target_roi"] = dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * 1.25 +dataframe["sell_roi"] = dataframe["&-s_close_mean"] - dataframe["&-s_close_std"] * 1.25 +``` + +If the user wishes to consider the population +of *historical predictions* for creating the dynamic target instead of the trained labels, (as discussed above) the user +can do so by setting `fit_live_prediction_candles` in the config to the number of historical prediction candles +the user wishes to use to generate target statistics. + +```json + "freqai": { + "fit_live_prediction_candles": 300, + } +``` + +If the user sets this value, FreqAI will initially use the predictions from the training data +and subsequently begin introducing real prediction data as it is generated. FreqAI will save +this historical data to be reloaded if the user stops and restarts a model with the same `identifier`. + +## Building a custom prediction model + +FreqAI has multiple example prediction model libraries, such as `Catboost` regression (`freqai/prediction_models/CatboostRegressor.py`) and `LightGBM` regression. +However, the user can customize and create their own prediction models using the `IFreqaiModel` class. +The user is encouraged to inherit `train()` and `predict()` to let them customize various aspects of their training procedures. + +## Feature engineering + +Features are added by the user inside the `populate_any_indicators()` method of the strategy +by prepending indicators with `%`, and labels with `&`. + +There are some important components/structures that the user *must* include when building their feature set; the use of these is shown below: + +```python + def populate_any_indicators( + self, pair, df, tf, informative=None, set_generalized_indicators=False + ): + """ + Function designed to automatically generate, name, and merge features + from user-indicated timeframes in the configuration file. The user controls the indicators + passed to the training/prediction by prepending indicators with `'%-' + coin ` + (see convention below). I.e., the user should not prepend any supporting metrics + (e.g., bb_lowerband below) with % unless they explicitly want to pass that metric to the + model. + :param pair: pair to be used as informative + :param df: strategy dataframe which will receive merges from informatives + :param tf: timeframe of the dataframe which will modify the feature names + :param informative: the dataframe associated with the informative pair + :param coin: the name of the coin which will modify the feature names. + """ + + coin = pair.split('/')[0] + + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) + + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + + bollinger = qtpylib.bollinger_bands( + qtpylib.typical_price(informative), window=t, stds=2.2 + ) + informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] + informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] + informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] + + informative[f"%-{coin}bb_width-period_{t}"] = ( + informative[f"{coin}bb_upperband-period_{t}"] + - informative[f"{coin}bb_lowerband-period_{t}"] + ) / informative[f"{coin}bb_middleband-period_{t}"] + informative[f"%-{coin}close-bb_lower-period_{t}"] = ( + informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] + ) + + informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative["volume"] / informative["volume"].rolling(t).mean() + ) + + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) + + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) + + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) + + return df +``` + +In the presented example strategy, the user does not wish to pass the `bb_lowerband` as a feature to the model, +and has therefore not prepended it with `%`. The user does, however, wish to pass `bb_width` to the +model for training/prediction and has therefore prepended it with `%`. + +The `include_timeframes` in the example config above are the timeframes (`tf`) of each call to `populate_any_indicators()` in the strategy. In the present case, the user is asking for the +`5m`, `15m`, and `4h` timeframes of the `rsi`, `mfi`, `roc`, and `bb_width` to be included in the feature set. + +The user can ask for each of the defined features to be included also from +informative pairs using the `include_corr_pairlist`. This means that the feature +set will include all the features from `populate_any_indicators` on all the `include_timeframes` for each of the correlated pairs defined in the config (`ETH/USD`, `LINK/USD`, and `BNB/USD`). + +`include_shifted_candles` indicates the number of previous +candles to include in the feature set. For example, `include_shifted_candles: 2` tells +FreqAI to include the past 2 candles for each of the features in the feature set. + +In total, the number of features the user of the presented example strat has created is: +length of `include_timeframes` * no. features in `populate_any_indicators()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles` + $= 3 * 3 * 3 * 2 * 2 = 108$. + +Another structure to consider is the location of the labels at the bottom of the example function (below `if set_generalized_indicators:`). +This is where the user will add single features and labels to their feature set to avoid duplication of them from +various configuration parameters that multiply the feature set, such as `include_timeframes`. + +!!! Note + Features **must** be defined in `populate_any_indicators()`. Definining FreqAI features in `populate_indicators()` + will cause the algorithm to fail in live/dry mode. If the user wishes to add generalized features that are not associated with + a specific pair or timeframe, they should use the following structure inside `populate_any_indicators()` + (as exemplified in `freqtrade/templates/FreqaiExampleStrategy.py`): + + ```python + def populate_any_indicators(self, metadata, pair, df, tf, informative=None, coin="", set_generalized_indicators=False): + + ... + + # Add generalized indicators here (because in live, it will call only this function to populate + # indicators for retraining). Notice how we ensure not to add them multiple times by associating + # these generalized indicators to the basepair/timeframe + if set_generalized_indicators: + df['%-day_of_week'] = (df["date"].dt.dayofweek + 1) / 7 + df['%-hour_of_day'] = (df['date'].dt.hour + 1) / 25 + + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) + ``` + + (Please see the example script located in `freqtrade/templates/FreqaiExampleStrategy.py` for a full example of `populate_any_indicators()`.) + +## Setting classifier targets + +FreqAI includes the `CatboostClassifier` via the flag `--freqaimodel CatboostClassifier`. The user should take care to set the classes using strings: + +```python +df['&s-up_or_down'] = np.where( df["close"].shift(-100) > df["close"], 'up', 'down') +``` + +Additionally, the example classifier models do not accommodate multiple labels, but they do allow multi-class classification within a single label column. + +## Running FreqAI + +There are two ways to train and deploy an adaptive machine learning model. FreqAI enables live deployment as well as backtesting analyses. In both cases, a model is trained periodically, as shown in the following figure. + +![freqai-window](assets/freqai_moving-window.jpg) + +### Running the model live + +FreqAI can be run dry/live using the following command: + +```bash +freqtrade trade --strategy FreqaiExampleStrategy --config config_freqai.example.json --freqaimodel LightGBMRegressor +``` + +By default, FreqAI will not find any existing models and will start by training a new one +based on the user's configuration settings. Following training, the model will be used to make predictions on incoming candles until a new model is available. New models are typically generated as often as possible, with FreqAI managing an internal queue of the coin pairs to try to keep all models equally up to date. FreqAI will always use the most recently trained model to make predictions on incoming live data. If the user does not want FreqAI to retrain new models as often as possible, they can set `live_retrain_hours` to tell FreqAI to wait at least that number of hours before training a new model. Additionally, the user can set `expired_hours` to tell FreqAI to avoid making predictions on models that are older than that number of hours. + +If the user wishes to start a dry/live run from a saved backtest model (or from a previously crashed dry/live session), the user only needs to reuse +the same `identifier` parameter: + +```json + "freqai": { + "identifier": "example", + "live_retrain_hours": 0.5 + } +``` + +In this case, although FreqAI will initiate with a +pre-trained model, it will still check to see how much time has elapsed since the model was trained, +and if a full `live_retrain_hours` has elapsed since the end of the loaded model, FreqAI will retrain. + +### Backtesting + +The FreqAI backtesting module can be executed with the following command: + +```bash +freqtrade backtesting --strategy FreqaiExampleStrategy --config config_freqai.example.json --freqaimodel LightGBMRegressor --timerange 20210501-20210701 +``` + +Backtesting mode requires the user to have the data pre-downloaded (unlike in dry/live mode where FreqAI automatically downloads the necessary data). The user should be careful to consider that the time range of the downloaded data is more than the backtesting time range. This is because FreqAI needs data prior to the desired backtesting time range in order to train a model to be ready to make predictions on the first candle of the user-set backtesting time range. More details on how to calculate the data to download can be found [here](#deciding-the-sliding-training-window-and-backtesting-duration). + +If this command has never been executed with the existing config file, it will train a new model +for each pair, for each backtesting window within the expanded `--timerange`. + +!!! Note "Model reuse" + Once the training is completed, the user can execute the backtesting again with the same config file and + FreqAI will find the trained models and load them instead of spending time training. This is useful + if the user wants to tweak (or even hyperopt) buy and sell criteria inside the strategy. If the user + *wants* to retrain a new model with the same config file, then they should simply change the `identifier`. + This way, the user can return to using any model they wish by simply specifying the `identifier`. + +--- + +### Deciding the size of the sliding training window and backtesting duration + +The user defines the backtesting timerange with the typical `--timerange` parameter in the +configuration file. The duration of the sliding training window is set by `train_period_days`, whilst +`backtest_period_days` is the sliding backtesting window, both in number of days (`backtest_period_days` can be +a float to indicate sub-daily retraining in live/dry mode). In the presented example config, +the user is asking FreqAI to use a training period of 30 days and backtest on the subsequent 7 days. +This means that if the user sets `--timerange 20210501-20210701`, +FreqAI will train have trained 8 separate models at the end of `--timerange` (because the full range comprises 8 weeks). After the training of the model, FreqAI will backtest the subsequent 7 days. The "sliding window" then moves one week forward (emulating FreqAI retraining once per week in live mode) and the new model uses the previous 30 days (including the 7 days used for backtesting by the previous model) to train. This is repeated until the end of `--timerange`. + +In live mode, the required training data is automatically computed and downloaded. However, in backtesting mode, +the user must manually enter the required number of `startup_candles` in the config. This value +is used to increase the data to FreqAI, which should be sufficient to enable all indicators +to be NaN free at the beginning of the first training. This is done by identifying the +longest timeframe (`4h` in presented example config) and the longest indicator period (`20` days in presented example config) +and adding this to the `train_period_days`. The units need to be in the base candle time frame: +`startup_candles` = ( 4 hours * 20 max period * 60 minutes/hour + 30 day train_period_days * 1440 minutes per day ) / 5 min (base time frame) = 9360. + +!!! Note + In dry/live mode, this is all precomputed and handled automatically. Thus, `startup_candle` has no influence on dry/live mode. + +!!! Note + Although fractional `backtest_period_days` is allowed, the user should be aware that the `--timerange` is divided by this value to determine the number of models that FreqAI will need to train in order to backtest the full range. For example, if the user wants to set a `--timerange` of 10 days, and asks for a `backtest_period_days` of 0.1, FreqAI will need to train 100 models per pair to complete the full backtest. Because of this, a true backtest of FreqAI adaptive training would take a *very* long time. The best way to fully test a model is to run it dry and let it constantly train. In this case, backtesting would take the exact same amount of time as a dry run. + +### Defining model expirations + +During dry/live mode, FreqAI trains each coin pair sequentially (on separate threads/GPU from the main Freqtrade bot). This means that there is always an age discrepancy between models. If a user is training on 50 pairs, and each pair requires 5 minutes to train, the oldest model will be over 4 hours old. This may be undesirable if the characteristic time scale (the trade duration target) for a strategy is less than 4 hours. The user can decide to only make trade entries if the model is less than +a certain number of hours old by setting the `expiration_hours` in the config file: + +```json + "freqai": { + "expiration_hours": 0.5, + } +``` + +In the presented example config, the user will only allow predictions on models that are less than 1/2 hours old. + +### Purging old model data + +FreqAI stores new model files each time it retrains. These files become obsolete as new models are trained and FreqAI adapts to new market conditions. Users planning to leave FreqAI running for extended periods of time with high frequency retraining should enable `purge_old_models` in their config: + +```json + "freqai": { + "purge_old_models": true, + } +``` + +This will automatically purge all models older than the two most recently trained ones. + +### Returning additional info from training + +The user may find that there are some important metrics that they'd like to return to the strategy at the end of each model training. +The user can include these metrics by assigning them to `dk.data['extra_returns_per_train']['my_new_value'] = XYZ` inside their custom prediction model class. FreqAI takes the `my_new_value` assigned in this dictionary and expands it to fit the return dataframe to the strategy. +The user can then use the value in the strategy with `dataframe['my_new_value']`. An example of how this is already used in FreqAI is +the `&*_mean` and `&*_std` values, which indicate the mean and standard deviation of the particular target (label) during the most recent training. +An example, where the user wants to use live metrics from the trade database, is shown below: + +```json + "freqai": { + "extra_returns_per_train": {"total_profit": 4} + } +``` + +The user needs to set the standard dictionary in the config so that FreqAI can return proper dataframe shapes. These values will likely be overridden by the prediction model, but in the case where the model has yet to set them, or needs a default initial value, this is the value that will be returned. + +### Setting up a follower + +The user can define: + +```json + "freqai": { + "follow_mode": true, + "identifier": "example" + } +``` + +to indicate to the bot that it should not train models, but instead should look for models trained by a leader with the same `identifier`. In this example, the user has a leader bot with the `identifier: "example"`. The leader bot is already running or launching simultaneously as the follower. +The follower will load models created by the leader and inference them to obtain predictions. + +## Data manipulation techniques + +### Feature normalization + +The feature set created by the user is automatically normalized to the training data. This includes all test data and unseen prediction data (dry/live/backtest). + +### Reducing data dimensionality with Principal Component Analysis + +Users can reduce the dimensionality of their features by activating the `principal_component_analysis` in the config: + +```json + "freqai": { + "feature_parameters" : { + "principal_component_analysis": true + } + } +``` + +This will perform PCA on the features and reduce the dimensionality of the data so that the explained variance of the data set is >= 0.999. + +### Stratifying the data for training and testing the model + +The user can stratify (group) the training/testing data using: + +```json + "freqai": { + "feature_parameters" : { + "stratify_training_data": 3 + } + } +``` + +This will split the data chronologically so that every Xth data point is used to test the model after training. In the +example above, the user is asking for every third data point in the dataframe to be used for +testing; the other points are used for training. + +The test data is used to evaluate the performance of the model after training. If the test score is high, the model is able to capture the behavior of the data well. If the test score is low, either the model either does not capture the complexity of the data, the test data is significantly different from the train data, or a different model should be used. + +### Controlling the model learning process + +Model training parameters are unique to the machine learning library selected by the user. FreqAI allows the user to set any parameter for any library using the `model_training_parameters` dictionary in the user configuration file. The example configuration file (found in `config_examples/config_freqai.example.json`) show some of the example parameters associated with `Catboost` and `LightGBM`, but the user can add any parameters available in those libraries. + +Data split parameters are defined in `data_split_parameters` which can be any parameters associated with `Sklearn`'s `train_test_split()` function. + +FreqAI includes some additional parameters such as `weight_factor`, which allows the user to weight more recent data more strongly +than past data via an exponential function: + +$$ W_i = \exp(\frac{-i}{\alpha*n}) $$ + +where $W_i$ is the weight of data point $i$ in a total set of $n$ data points. Below is a figure showing the effect of different weight factors on the data points (candles) in a feature set. + +![weight-factor](assets/freqai_weight-factor.jpg) + +`train_test_split()` has a parameters called `shuffle` that allows the user to keep the data unshuffled. This is particularly useful to avoid biasing training with temporally auto-correlated data. + +Finally, `label_period_candles` defines the offset (number of candles into the future) used for the `labels`. In the presented example config, +the user is asking for `labels` that are 24 candles in the future. + +### Outlier removal + +#### Removing outliers with the Dissimilarity Index + +The user can tell FreqAI to remove outlier data points from the training/test data sets using a Dissimilarity Index by including the following statement in the config: + +```json + "freqai": { + "feature_parameters" : { + "DI_threshold": 1 + } + } +``` + +Equity and crypto markets suffer from a high level of non-patterned noise in the form of outlier data points. The Dissimilarity Index (DI) aims to quantify the uncertainty associated with each prediction made by the model. The DI allows predictions which are outliers (not existent in the model feature space) to be thrown out due to low levels of certainty. + +To do so, FreqAI measures the distance between each training data point (feature vector), $X_{a}$, and all other training data points: + +$$ d_{ab} = \sqrt{\sum_{j=1}^p(X_{a,j}-X_{b,j})^2} $$ + +where $d_{ab}$ is the distance between the normalized points $a$ and $b$. $p$ is the number of features, i.e., the length of the vector $X$. The characteristic distance, $\overline{d}$ for a set of training data points is simply the mean of the average distances: + +$$ \overline{d} = \sum_{a=1}^n(\sum_{b=1}^n(d_{ab}/n)/n) $$ + +$\overline{d}$ quantifies the spread of the training data, which is compared to the distance between a new prediction feature vectors, $X_k$ and all the training data: + +$$ d_k = \arg \min d_{k,i} $$ + +which enables the estimation of the Dissimilarity Index as: + +$$ DI_k = d_k/\overline{d} $$ + +The user can tweak the DI through the `DI_threshold` to increase or decrease the extrapolation of the trained model. + +Below is a figure that describes the DI for a 3D data set. + +![DI](assets/freqai_DI.jpg) + +#### Removing outliers using a Support Vector Machine (SVM) + +The user can tell FreqAI to remove outlier data points from the training/test data sets using a SVM by setting: + +```json + "freqai": { + "feature_parameters" : { + "use_SVM_to_remove_outliers": true + } + } +``` + +FreqAI will train an SVM on the training data (or components of it if the user activated +`principal_component_analysis`) and remove any data point that the SVM deems to be beyond the feature space. + +The parameter `shuffle` is by default set to `False` to ensure consistent results. If it is set to `True`, running the SVM multiple times on the same data set might result in different outcomes due to `max_iter` being to low for the algorithm to reach the demanded `tol`. Increasing `max_iter` solves this issue but causes the procedure to take longer time. + +The parameter `nu`, *very* broadly, is the amount of data points that should be considered outliers. + +#### Removing outliers with DBSCAN + +The user can configure FreqAI to use DBSCAN to cluster and remove outliers from the training/test data set or incoming outliers from predictions, by activating `use_DBSCAN_to_remove_outliers` in the config: + +```json + "freqai": { + "feature_parameters" : { + "use_DBSCAN_to_remove_outliers": true + } + } +``` + +DBSCAN is an unsupervised machine learning algorithm that clusters data without needing to know how many clusters there should be. + +Given a number of data points $N$, and a distance $\varepsilon$, DBSCAN clusters the data set by setting all data points that have $N-1$ other data points within a distance of $\varepsilon$ as *core points*. A data point that is within a distance of $\varepsilon$ from a *core point* but that does not have $N-1$ other data points within a distance of $\varepsilon$ from itself is considered an *edge point*. A cluster is then the collection of *core points* and *edge points*. Data points that have no other data points at a distance $<\varepsilon$ are considered outliers. The figure below shows a cluster with $N = 3$. + +![dbscan](assets/freqai_dbscan.jpg) + +FreqAI uses `sklearn.cluster.DBSCAN` (details are available on scikit-learn's webpage [here](#https://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html)) with `min_samples` ($N$) taken as double the no. of user-defined features, and `eps` ($\varepsilon$) taken as the longest distance in the *k-distance graph* computed from the nearest neighbors in the pairwise distances of all data points in the feature set. + +## Additional information + +### Common pitfalls + +FreqAI cannot be combined with dynamic `VolumePairlists` (or any pairlist filter that adds and removes pairs dynamically). +This is for performance reasons - FreqAI relies on making quick predictions/retrains. To do this effectively, +it needs to download all the training data at the beginning of a dry/live instance. FreqAI stores and appends +new candles automatically for future retrains. This means that if new pairs arrive later in the dry run due to a volume pairlist, it will not have the data ready. However, FreqAI does work with the `ShufflePairlist` or a `VolumePairlist` which keeps the total pairlist constant (but reorders the pairs according to volume). + +## Credits + +FreqAI was developed by a group of individuals who all contributed specific skillsets to the project. + +Conception and software development: +Robert Caulk @robcaulk + +Theoretical brainstorming, data analysis: +Elin Törnquist @th0rntwig + +Code review, software architecture brainstorming: +@xmatthias + +Beta testing and bug reporting: +@bloodhunter4rc, Salah Lamkadem @ikonx, @ken11o2, @longyu, @paranoidandy, @smidelis, @smarm +Juha Nykänen @suikula, Wagner Costa @wagnercosta diff --git a/docs/hyperopt.md b/docs/hyperopt.md index c9ec30056..6b6c2a772 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -40,7 +40,8 @@ pip install -r requirements-hyperopt.txt ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] - [--recursive-strategy-search] [-i TIMEFRAME] + [--recursive-strategy-search] [--freqaimodel NAME] + [--freqaimodel-path PATH] [-i TIMEFRAME] [--timerange TIMERANGE] [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] @@ -53,7 +54,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] [--hyperopt-loss NAME] [--disable-param-export] - [--ignore-missing-spaces] + [--ignore-missing-spaces] [--analyze-per-epoch] optional arguments: -h, --help show this help message and exit @@ -129,6 +130,7 @@ optional arguments: --ignore-missing-spaces, --ignore-unparameterized-spaces Suppress errors for any requested Hyperopt spaces that do not contain any parameters. + --analyze-per-epoch Run populate_indicators once per epoch. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -154,6 +156,10 @@ Strategy arguments: --recursive-strategy-search Recursively search for a strategy in the strategies folder. + --freqaimodel NAME Specify a custom freqaimodels. + --freqaimodel-path PATH + Specify additional lookup path for freqaimodels. + ``` ### Hyperopt checklist @@ -185,7 +191,7 @@ Rarely you may also need to create a [nested class](advanced-hyperopt.md#overrid ### Hyperopt execution logic -Hyperopt will first load your data into memory and will then run `populate_indicators()` once per Pair to generate all indicators. +Hyperopt will first load your data into memory and will then run `populate_indicators()` once per Pair to generate all indicators, unless `--analyze-per-epoch` is specified. Hyperopt will then spawn into different processes (number of processors, or `-j `), and run backtesting over and over again, changing the parameters that are part of the `--spaces` defined. @@ -426,9 +432,10 @@ While this strategy is most likely too simple to provide consistent profit, it s `range` property may also be used with `DecimalParameter` and `CategoricalParameter`. `RealParameter` does not provide this property due to infinite search space. ??? Hint "Performance tip" - By doing the calculation of all possible indicators in `populate_indicators()`, the calculation of the indicator happens only once for every parameter. - While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values). - You should however try to use space ranges as small as possible. Every new column will require more memory, and every possibility hyperopt can try will increase the search space. + During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value. This will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues. + + In either case, you should try to use space ranges as small as possible this will improve CPU/RAM usage in both scenarios. + ## Optimizing protections @@ -879,6 +886,7 @@ To combat these, you have multiple options: * Avoid using `--timeframe-detail` (this loads a lot of additional data into memory). * Reduce the number of parallel processes (`-j `). * Increase the memory of your machine. +* Use `--analyze-per-epoch` if you're using a lot of parameters with `.range` functionality. ## The objective has been evaluated at this point before. diff --git a/docs/installation.md b/docs/installation.md index 92aa59498..9dd14274a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -326,6 +326,16 @@ python3 -m pip install --upgrade pip python3 -m pip install -e . ``` +Patch conda libta-lib (Linux only) + +```bash +# Ensure that the environment is active! +conda activate freqtrade-conda + +cd build_helpers +bash install_ta-lib.sh ${CONDA_PREFIX} nosudo +``` + ### Congratulations [You are ready](#you-are-ready), and run the bot diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 205516d6d..bffc04d1c 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.3.1 -mkdocs-material==8.3.9 +mkdocs-material==8.4.1 mdx_truly_sane_lists==1.3 pymdown-extensions==9.5 jinja2==3.1.2 diff --git a/docs/rest-api.md b/docs/rest-api.md index 1ec9b6c12..cc82aadda 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -163,6 +163,8 @@ python3 scripts/rest_client.py --config rest_config.json [optional par | `strategy ` | Get specific Strategy content. **Alpha** | `available_pairs` | List available backtest data. **Alpha** | `version` | Show version. +| `sysinfo` | Show informations about the system load. +| `health` | Show bot health (last bot loop). !!! Warning "Alpha status" Endpoints labeled with *Alpha status* above may change at any time without notice. @@ -227,6 +229,11 @@ forceexit Force-exit a trade. :param tradeid: Id of the trade (can be received via status command) + :param ordertype: Order type to use (must be market or limit) + :param amount: Amount to sell. Full sell if not given + +health + Provides a quick health check of the running bot. locks Return current locks @@ -312,12 +319,13 @@ version whitelist Show the current whitelist. + ``` ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. -This will enable the Swagger UI at the `/docs` endpoint. By default, that's running at http://localhost:8080/docs/ - but it'll depend on your settings. +This will enable the Swagger UI at the `/docs` endpoint. By default, that's running at http://localhost:8080/docs - but it'll depend on your settings. ### Advanced API usage using JWT tokens diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index f584bd1bb..0b8403414 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -75,7 +75,7 @@ class AwesomeStrategy(IStrategy): ``` -### Stake size management +## Stake size management Called before entering a trade, makes it possible to manage your position size when placing a new trade. @@ -423,7 +423,7 @@ class AwesomeStrategy(IStrategy): !!! Warning "Backtesting" Custom prices are supported in backtesting (starting with 2021.12), and orders will fill if the price falls within the candle's low/high range. Orders that don't fill immediately are subject to regular timeout handling, which happens once per (detail) candle. - `custom_exit_price()` is only called for sells of type exit_signal and Custom exit. All other exit-types will use regular backtesting prices. + `custom_exit_price()` is only called for sells of type exit_signal, Custom exit and partial exits. All other exit-types will use regular backtesting prices. ## Custom order timeout rules @@ -623,12 +623,13 @@ class AwesomeStrategy(IStrategy): !!! Warning `confirm_trade_exit()` can prevent stoploss exits, causing significant losses as this would ignore stoploss exits. + `confirm_trade_exit()` will not be called for Liquidations - as liquidations are forced by the exchange, and therefore cannot be rejected. ## Adjust trade position The `position_adjustment_enable` strategy property enables the usage of `adjust_trade_position()` callback in the strategy. For performance reasons, it's disabled by default and freqtrade will show a warning message on startup if enabled. -`adjust_trade_position()` can be used to perform additional orders, for example to manage risk with DCA (Dollar Cost Averaging). +`adjust_trade_position()` can be used to perform additional orders, for example to manage risk with DCA (Dollar Cost Averaging) or to increase or decrease positions. `max_entry_position_adjustment` property is used to limit the number of additional buys per trade (on top of the first buy) that the bot can execute. By default, the value is -1 which means the bot have no limit on number of adjustment buys. @@ -636,10 +637,13 @@ The strategy is expected to return a stake_amount (in stake currency) between `m If there are not enough funds in the wallet (the return value is above `max_stake`) then the signal will be ignored. Additional orders also result in additional fees and those orders don't count towards `max_open_trades`. -This callback is **not** called when there is an open order (either buy or sell) waiting for execution, or when you have reached the maximum amount of extra buys that you have set on `max_entry_position_adjustment`. +This callback is **not** called when there is an open order (either buy or sell) waiting for execution. + `adjust_trade_position()` is called very frequently for the duration of a trade, so you must keep your implementation as performant as possible. -Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position, no matter if it's a long or short trade. Modifications to leverage are not possible. +Additional Buys are ignored once you have reached the maximum amount of extra buys that you have set on `max_entry_position_adjustment`, but the callback is called anyway looking for partial exits. + +Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position (negative values will decrease your position), no matter if it's a long or short trade. Modifications to leverage are not possible. !!! Note "About stake size" Using fixed stake size means it will be the amount used for the first order, just like without position adjustment. @@ -648,12 +652,12 @@ Position adjustments will always be applied in the direction of the trade, so a !!! Warning Stoploss is still calculated from the initial opening price, not averaged price. + Regular stoploss rules still apply (cannot move down). -!!! Warning "/stopbuy" - While `/stopbuy` command stops the bot from entering new trades, the position adjustment feature will continue buying new orders on existing trades. + While `/stopentry` command stops the bot from entering new trades, the position adjustment feature will continue buying new orders on existing trades. !!! Warning "Backtesting" - During backtesting this callback is called for each candle in `timeframe` or `timeframe_detail`, so performance will be affected. + During backtesting this callback is called for each candle in `timeframe` or `timeframe_detail`, so run-time performance will be affected. ``` python from freqtrade.persistence import Trade @@ -674,7 +678,7 @@ class DigDeeperStrategy(IStrategy): max_dca_multiplier = 5.5 # This is called when placing the initial order (opening trade) -def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: float, + def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: float, proposed_stake: float, min_stake: Optional[float], max_stake: float, leverage: float, entry_tag: Optional[str], side: str, **kwargs) -> float: @@ -684,22 +688,41 @@ def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: f return proposed_stake / self.max_dca_multiplier def adjust_trade_position(self, trade: Trade, current_time: datetime, - current_rate: float, current_profit: float, min_stake: Optional[float], - max_stake: float, **kwargs): + current_rate: float, current_profit: float, + min_stake: Optional[float], max_stake: float, + current_entry_rate: float, current_exit_rate: float, + current_entry_profit: float, current_exit_profit: float, + **kwargs) -> Optional[float]: """ - Custom trade adjustment logic, returning the stake amount that a trade should be increased. - This means extra buy orders with additional fees. + Custom trade adjustment logic, returning the stake amount that a trade should be + increased or decreased. + This means extra buy or sell orders with additional fees. + Only called when `position_adjustment_enable` is set to True. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, returns None :param trade: trade object. :param current_time: datetime object, containing the current datetime :param current_rate: Current buy rate. :param current_profit: Current profit (as ratio), calculated based on current_rate. - :param min_stake: Minimal stake size allowed by exchange. - :param max_stake: Balance available for trading. + :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) + :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). + :param current_entry_rate: Current rate using entry pricing. + :param current_exit_rate: Current rate using exit pricing. + :param current_entry_profit: Current profit using entry pricing. + :param current_exit_profit: Current profit using exit pricing. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: Stake amount to adjust your trade + :return float: Stake amount to adjust your trade, + Positive values to increase position, Negative values to decrease position. + Return None for no action. """ + if current_profit > 0.05 and trade.nr_of_successful_exits == 0: + # Take half of the profit at +5% + return -(trade.stake_amount / 2) + if current_profit > -0.05: return None @@ -734,6 +757,25 @@ def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: f ``` +### Position adjust calculations + +* Entry rates are calculated using weighted averages. +* Exits will not influence the average entry rate. +* Partial exit relative profit is relative to the average entry price at this point. +* Final exit relative profit is calculated based on the total invested capital. (See example below) + +??? example "Calculation example" + *This example assumes 0 fees for simplicity, and a long position on an imaginary coin.* + + * Buy 100@8\$ + * Buy 100@9\$ -> Avg price: 8.5\$ + * Sell 100@10\$ -> Avg price: 8.5\$, realized profit 150\$, 17.65% + * Buy 150@11\$ -> Avg price: 10\$, realized profit 150\$, 17.65% + * Sell 100@12\$ -> Avg price: 10\$, total realized profit 350\$, 20% + * Sell 150@14\$ -> Avg price: 10\$, total realized profit 950\$, 40% + + The total profit for this trade was 950$ on a 3350$ investment (`100@8$ + 100@9$ + 150@11$`). As such - the final relative profit is 28.35% (`950 / 3350`). + ## Adjust Entry Price The `adjust_entry_price()` callback may be used by strategy developer to refresh/replace limit orders upon arrival of new candles. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 78256e0ee..260e253c4 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -617,9 +617,8 @@ Please always check the mode of operation to select the correct method to get da ### *available_pairs* ``` python -if self.dp: - for pair, timeframe in self.dp.available_pairs: - print(f"available {pair}, {timeframe}") +for pair, timeframe in self.dp.available_pairs: + print(f"available {pair}, {timeframe}") ``` ### *current_whitelist()* @@ -630,7 +629,7 @@ The strategy might look something like this: *Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day RSI to buy and sell.* -Due to the limited available data, it's very difficult to resample `5m` candles into daily candles for use in a 14 day RSI. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least! +Due to the limited available data, it's very difficult to resample `5m` candles into daily candles for use in a 14 day RSI. Most exchanges limit us to just 500-1000 candles which effectively gives us around 1.74 daily candles. We need 14 days at least! Since we can't resample the data we will have to use an informative pair; and since the whitelist will be dynamic we don't know which pair(s) to use. @@ -653,10 +652,9 @@ This is where calling `self.dp.current_whitelist()` comes in handy. ``` python # fetch live / historical candle (OHLCV) data for the first informative pair -if self.dp: - inf_pair, inf_timeframe = self.informative_pairs()[0] - informative = self.dp.get_pair_dataframe(pair=inf_pair, - timeframe=inf_timeframe) +inf_pair, inf_timeframe = self.informative_pairs()[0] +informative = self.dp.get_pair_dataframe(pair=inf_pair, + timeframe=inf_timeframe) ``` !!! Warning "Warning about backtesting" @@ -671,10 +669,9 @@ It can also be used in specific callbacks to get the signal that caused the acti ``` python # fetch current dataframe -if self.dp: - if self.dp.runmode.value in ('live', 'dry_run'): - dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'], - timeframe=self.timeframe) +if self.dp.runmode.value in ('live', 'dry_run'): + dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'], + timeframe=self.timeframe) ``` !!! Note "No data available" @@ -684,11 +681,10 @@ if self.dp: ### *orderbook(pair, maximum)* ``` python -if self.dp: - if self.dp.runmode.value in ('live', 'dry_run'): - ob = self.dp.orderbook(metadata['pair'], 1) - dataframe['best_bid'] = ob['bids'][0][0] - dataframe['best_ask'] = ob['asks'][0][0] +if self.dp.runmode.value in ('live', 'dry_run'): + ob = self.dp.orderbook(metadata['pair'], 1) + dataframe['best_bid'] = ob['bids'][0][0] + dataframe['best_ask'] = ob['asks'][0][0] ``` The orderbook structure is aligned with the order structure from [ccxt](https://github.com/ccxt/ccxt/wiki/Manual#order-book-structure), so the result will look as follows: @@ -717,12 +713,11 @@ Therefore, using `ob['bids'][0][0]` as demonstrated above will result in using t ### *ticker(pair)* ``` python -if self.dp: - if self.dp.runmode.value in ('live', 'dry_run'): - ticker = self.dp.ticker(metadata['pair']) - dataframe['last_price'] = ticker['last'] - dataframe['volume24h'] = ticker['quoteVolume'] - dataframe['vwap'] = ticker['vwap'] +if self.dp.runmode.value in ('live', 'dry_run'): + ticker = self.dp.ticker(metadata['pair']) + dataframe['last_price'] = ticker['last'] + dataframe['volume24h'] = ticker['quoteVolume'] + dataframe['vwap'] = ticker['vwap'] ``` !!! Warning @@ -732,7 +727,24 @@ if self.dp: data returned from the exchange and add appropriate error handling / defaults. !!! Warning "Warning about backtesting" - This method will always return up-to-date values - so usage during backtesting / hyperopt will lead to wrong results. + This method will always return up-to-date values - so usage during backtesting / hyperopt without runmode checks will lead to wrong results. + +### Send Notification + +The dataprovider `.send_msg()` function allows you to send custom notifications from your strategy. +Identical notifications will only be sent once per candle, unless the 2nd argument (`always_send`) is set to True. + +``` python + self.dp.send_msg(f"{metadata['pair']} just got hot!") + + # Force send this notification, avoid caching (Please read warning below!) + self.dp.send_msg(f"{metadata['pair']} just got hot!", always_send=True) +``` + +Notifications will only be sent in trading modes (Live/Dry-run) - so this method can be called without conditions for backtesting. + +!!! Warning "Spamming" + You can spam yourself pretty good by setting `always_send=True` in this method. Use this with great care and only in conditions you know will not happen throughout a candle to avoid a message every 5 seconds. ### Complete Data-provider sample diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index fbfce37d1..1526ea038 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -14,7 +14,7 @@ from freqtrade.configuration import Configuration # Initialize empty configuration object config = Configuration.from_files([]) -# Optionally, use existing configuration file +# Optionally (recommended), use existing configuration file # config = Configuration.from_files(["config.json"]) # Define some constants @@ -22,7 +22,7 @@ config["timeframe"] = "5m" # Name of the strategy class config["strategy"] = "SampleStrategy" # Location of the data -data_location = Path(config['user_data_dir'], 'data', 'binance') +data_location = config['datadir'] # Pair to analyze - Only use one pair here pair = "BTC/USDT" ``` diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 9853e15c6..ece8700de 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -98,6 +98,7 @@ Example configuration showing the different settings: "exit_fill": "off", "protection_trigger": "off", "protection_trigger_global": "on", + "strategy_msg": "off", "show_candle": "off" }, "reload": true, @@ -109,7 +110,8 @@ Example configuration showing the different settings: `exit` notifications are sent when the order is placed, while `exit_fill` notifications are sent when the order is filled on the exchange. `*_fill` notifications are off by default and must be explicitly enabled. `protection_trigger` notifications are sent when a protection triggers and `protection_trigger_global` notifications trigger when global protections are triggered. -`show_candle` - show candle values as part of entry/exit messages. Only possible value is "ohlc". +`strategy_msg` - Receive notifications from the strategy, sent via `self.dp.send_msg()` from the strategy [more details](strategy-customization.md#send-notification). +`show_candle` - show candle values as part of entry/exit messages. Only possible values are `"ohlc"` or `"off"`. `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown. `reload` allows you to disable reload-buttons on selected messages. @@ -147,7 +149,7 @@ You can create your own keyboard in `config.json`: !!! Note "Supported Commands" Only the following commands are allowed. Command arguments are not supported! - `/start`, `/stop`, `/status`, `/status table`, `/trades`, `/profit`, `/performance`, `/daily`, `/stats`, `/count`, `/locks`, `/balance`, `/stopbuy`, `/reload_config`, `/show_config`, `/logs`, `/whitelist`, `/blacklist`, `/edge`, `/help`, `/version` + `/start`, `/stop`, `/status`, `/status table`, `/trades`, `/profit`, `/performance`, `/daily`, `/stats`, `/count`, `/locks`, `/balance`, `/stopentry`, `/reload_config`, `/show_config`, `/logs`, `/whitelist`, `/blacklist`, `/edge`, `/help`, `/version` ## Telegram commands @@ -159,7 +161,7 @@ official commands. You can ask at any moment for help with `/help`. |----------|-------------| | `/start` | Starts the trader | `/stop` | Stops the trader -| `/stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules. +| `/stopbuy | /stopentry` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules. | `/reload_config` | Reloads the configuration file | `/show_config` | Shows part of the current configuration with relevant settings to operation | `/logs [limit]` | Show last log messages. @@ -185,7 +187,7 @@ official commands. You can ask at any moment for help with `/help`. | `/stats` | Shows Wins / losses by Exit reason as well as Avg. holding durations for buys and sells | `/exits` | Shows Wins / losses by Exit reason as well as Avg. holding durations for buys and sells | `/entries` | Shows Wins / losses by Exit reason as well as Avg. holding durations for buys and sells -| `/whitelist` | Show the current whitelist +| `/whitelist [sorted] [baseonly]` | Show the current whitelist. Optionally display in alphabetical order and/or with just the base currency of each pairing. | `/blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist. | `/edge` | Show validated pairs by Edge if it is enabled. | `/help` | Show help message diff --git a/docs/utils.md b/docs/utils.md index 0dd88b242..5646365e4 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -611,6 +611,26 @@ Common arguments: ``` +### Webserver mode - docker + +You can also use webserver mode via docker. +Starting a one-off container requires the configuration of the port explicitly, as ports are not exposed by default. +You can use `docker-compose run --rm -p 127.0.0.1:8080:8080 freqtrade webserver` to start a one-off container that'll be removed once you stop it. This assumes that port 8080 is still available and no other bot is running on that port. + +Alternatively, you can reconfigure the docker-compose file to have the command updated: + +``` yml + command: > + webserver + --config /freqtrade/user_data/config.json +``` + +You can now use `docker-compose up` to start the webserver. +This assumes that the configuration has a webserver enabled and configured for docker (listening port = `0.0.0.0`). + +!!! Tip + Don't forget to reset the command back to the trade command if you want to start a live or dry-run bot. + ## Show previous Backtest results Allows you to show previous backtest results. diff --git a/environment.yml b/environment.yml index 19f3c7f5a..d6d85de9d 100644 --- a/environment.yml +++ b/environment.yml @@ -9,6 +9,7 @@ dependencies: - pandas - pip + - py-find-1st - aiohttp - SQLAlchemy - python-telegram-bot @@ -64,7 +65,7 @@ dependencies: - pip: - pycoingecko - - py_find_1st + # - py_find_1st - tables - pytest-random-order - ccxt diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 21054a723..6c5c52a04 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.7' +__version__ = '2022.8' if 'dev' in __version__: try: diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index fc3eda14d..37ce17f21 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -12,7 +12,8 @@ from freqtrade.constants import DEFAULT_CONFIG ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"] -ARGS_STRATEGY = ["strategy", "strategy_path", "recursive_strategy_search"] +ARGS_STRATEGY = ["strategy", "strategy_path", "recursive_strategy_search", "freqaimodel", + "freqaimodel_path"] ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"] @@ -33,7 +34,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", "hyperopt_loss", "disableparamexport", - "hyperopt_ignore_missing_space"] + "hyperopt_ignore_missing_space", "analyze_per_epoch"] ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] @@ -68,7 +69,7 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "exchange", "tradin ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] -ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"] +ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive", "timerange", "download_trades", "exchange", "timeframes", diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index be881c8ed..01cfa800a 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -67,7 +67,7 @@ def ask_user_config() -> Dict[str, Any]: "type": "text", "name": "stake_amount", "message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):", - "default": "100", + "default": "unlimited", "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val), "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"' if val == UNLIMITED_STAKE_AMOUNT @@ -164,7 +164,7 @@ def ask_user_config() -> Dict[str, Any]: "when": lambda x: x['telegram'] }, { - "type": "text", + "type": "password", "name": "telegram_chat_id", "message": "Insert Telegram chat id", "when": lambda x: x['telegram'] @@ -191,7 +191,7 @@ def ask_user_config() -> Dict[str, Any]: "when": lambda x: x['api_server'] }, { - "type": "text", + "type": "password", "name": "api_server_password", "message": "Insert api-server password", "when": lambda x: x['api_server'] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 3370ce64b..3d094da36 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -255,6 +255,13 @@ AVAILABLE_CLI_OPTIONS = { nargs='+', default='default', ), + "analyze_per_epoch": Arg( + '--analyze-per-epoch', + help='Run populate_indicators once per epoch.', + action='store_true', + default=False, + ), + "print_all": Arg( '--print-all', help='Print all results, not only the best ones.', @@ -367,7 +374,7 @@ AVAILABLE_CLI_OPTIONS = { metavar='BASE_CURRENCY', ), "trading_mode": Arg( - '--trading-mode', + '--trading-mode', '--tradingmode', help='Select Trading mode', choices=constants.TRADING_MODES, ), @@ -434,6 +441,11 @@ AVAILABLE_CLI_OPTIONS = { help='Storage format for downloaded trades data. (default: `jsongz`).', choices=constants.AVAILABLE_DATAHANDLERS, ), + "show_timerange": Arg( + '--show-timerange', + help='Show timerange available for available data. (May take a while to calculate).', + action='store_true', + ), "exchange": Arg( '--exchange', help=f'Exchange name (default: `{constants.DEFAULT_EXCHANGE}`). ' @@ -450,7 +462,7 @@ AVAILABLE_CLI_OPTIONS = { ), "prepend_data": Arg( '--prepend', - help='Allow data prepending.', + help='Allow data prepending. (Data-appending is disabled)', action='store_true', ), "erase": Arg( @@ -647,4 +659,14 @@ AVAILABLE_CLI_OPTIONS = { nargs='+', default=[], ), + "freqaimodel": Arg( + '--freqaimodel', + help='Specify a custom freqaimodels.', + metavar='NAME', + ), + "freqaimodel_path": Arg( + '--freqaimodel-path', + help='Specify additional lookup path for freqaimodels.', + metavar='PATH', + ), } diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 61a99782e..360387aa6 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -5,14 +5,14 @@ from datetime import datetime, timedelta from typing import Any, Dict, List from freqtrade.configuration import TimeRange, setup_utils_configuration +from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) from freqtrade.enums import CandleType, RunMode, TradingMode from freqtrade.exceptions import OperationalException -from freqtrade.exchange import timeframe_to_minutes -from freqtrade.exchange.exchange import market_is_active -from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist +from freqtrade.exchange import market_is_active, timeframe_to_minutes +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist from freqtrade.resolvers import ExchangeResolver @@ -50,7 +50,8 @@ def start_download_data(args: Dict[str, Any]) -> None: exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) markets = [p for p, m in exchange.markets.items() if market_is_active(m) or config.get('include_inactive')] - expanded_pairs = expand_pairlist(config['pairs'], markets) + + expanded_pairs = dynamic_expand_pairlist(config, markets) # Manual validations of relevant settings if not config['exchange'].get('skip_pair_validation', False): @@ -79,7 +80,7 @@ def start_download_data(args: Dict[str, Any]) -> None: data_format_trades=config['dataformat_trades'], ) else: - if not exchange._ft_has.get('ohlcv_has_history', True): + if not exchange.get_option('ohlcv_has_history', True): raise OperationalException( f"Historic klines not available for {exchange.name}. " "Please use `--dl-trades` instead for this exchange " @@ -176,17 +177,31 @@ def start_list_data(args: Dict[str, Any]) -> None: paircombs = [comb for comb in paircombs if comb[0] in args['pairs']] print(f"Found {len(paircombs)} pair / timeframe combinations.") - groupedpair = defaultdict(list) - for pair, timeframe, candle_type in sorted( - paircombs, - key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]) - ): - groupedpair[(pair, candle_type)].append(timeframe) + if not config.get('show_timerange'): + groupedpair = defaultdict(list) + for pair, timeframe, candle_type in sorted( + paircombs, + key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]) + ): + groupedpair[(pair, candle_type)].append(timeframe) - if groupedpair: + if groupedpair: + print(tabulate([ + (pair, ', '.join(timeframes), candle_type) + for (pair, candle_type), timeframes in groupedpair.items() + ], + headers=("Pair", "Timeframe", "Type"), + tablefmt='psql', stralign='right')) + else: + paircombs1 = [( + pair, timeframe, candle_type, + *dhc.ohlcv_data_min_max(pair, timeframe, candle_type) + ) for pair, timeframe, candle_type in paircombs] print(tabulate([ - (pair, ', '.join(timeframes), candle_type) - for (pair, candle_type), timeframes in groupedpair.items() - ], - headers=("Pair", "Timeframe", "Type"), + (pair, timeframe, candle_type, + start.strftime(DATETIME_PRINT_FORMAT), + end.strftime(DATETIME_PRINT_FORMAT)) + for pair, timeframe, candle_type, start, end in paircombs1 + ], + headers=("Pair", "Timeframe", "Type", 'From', 'To'), tablefmt='psql', stralign='right')) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index cf41c0ca9..730a4e47f 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -4,5 +4,4 @@ from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.configuration.configuration import Configuration -from freqtrade.configuration.PeriodicCache import PeriodicCache from freqtrade.configuration.timerange import TimeRange diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index d46d54cb0..7c68ac46c 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -97,6 +97,8 @@ class Configuration: self._process_analyze_options(config) + self._process_freqai_options(config) + # Check if the exchange set by the user is supported check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) @@ -300,6 +302,9 @@ class Configuration: self._args_to_config(config, argname='spaces', logstring='Parameter -s/--spaces detected: {}') + self._args_to_config(config, argname='analyze_per_epoch', + logstring='Parameter --analyze-per-epoch detected.') + self._args_to_config(config, argname='print_all', logstring='Parameter --print-all detected ...') @@ -424,6 +429,9 @@ class Configuration: self._args_to_config(config, argname='dataformat_trades', logstring='Using "{}" to store trades data.') + self._args_to_config(config, argname='show_timerange', + logstring='Detected --show-timerange') + def _process_data_options(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='new_pairs_days', logstring='Detected --new-pairs-days: {}') @@ -461,6 +469,16 @@ class Configuration: config.update({'runmode': self.runmode}) + def _process_freqai_options(self, config: Dict[str, Any]) -> None: + + self._args_to_config(config, argname='freqaimodel', + logstring='Using freqaimodel class name: {}') + + self._args_to_config(config, argname='freqaimodel_path', + logstring='Using freqaimodel path: {}') + + return + def _args_to_config(self, config: Dict[str, Any], argname: str, logstring: str, logfun: Optional[Callable] = None, deprecated_msg: Optional[str] = None) -> None: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 6d74ceafd..ddbc84fa9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -55,6 +55,7 @@ FTHYPT_FILEVERSION = 'fthypt_fileversion' USERPATH_HYPEROPTS = 'hyperopts' USERPATH_STRATEGIES = 'strategies' USERPATH_NOTEBOOKS = 'notebooks' +USERPATH_FREQAIMODELS = 'freqaimodels' TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent'] WEBHOOK_FORMAT_OPTIONS = ['form', 'json', 'raw'] @@ -240,6 +241,7 @@ CONF_SCHEMA = { }, 'exchange': {'$ref': '#/definitions/exchange'}, 'edge': {'$ref': '#/definitions/edge'}, + 'freqai': {'$ref': '#/definitions/freqai'}, 'experimental': { 'type': 'object', 'properties': { @@ -317,6 +319,10 @@ CONF_SCHEMA = { 'type': 'string', 'enum': ['off', 'ohlc'], }, + 'strategy_msg': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + }, } }, 'reload': {'type': 'boolean'}, @@ -476,7 +482,60 @@ CONF_SCHEMA = { 'remove_pumps': {'type': 'boolean'} }, 'required': ['process_throttle_secs', 'allowed_risk'] - } + }, + "freqai": { + "type": "object", + "properties": { + "enabled": {"type": "boolean", "default": False}, + "keras": {"type": "boolean", "default": False}, + "conv_width": {"type": "integer", "default": 2}, + "train_period_days": {"type": "integer", "default": 0}, + "backtest_period_days": {"type": "number", "default": 7}, + "identifier": {"type": "string", "default": "example"}, + "feature_parameters": { + "type": "object", + "properties": { + "include_corr_pairlist": {"type": "array"}, + "include_timeframes": {"type": "array"}, + "label_period_candles": {"type": "integer"}, + "include_shifted_candles": {"type": "integer", "default": 0}, + "DI_threshold": {"type": "number", "default": 0}, + "weight_factor": {"type": "number", "default": 0}, + "principal_component_analysis": {"type": "boolean", "default": False}, + "use_SVM_to_remove_outliers": {"type": "boolean", "default": False}, + "svm_params": {"type": "object", + "properties": { + "shuffle": {"type": "boolean", "default": False}, + "nu": {"type": "number", "default": 0.1} + }, + } + }, + "required": ["include_timeframes", "include_corr_pairlist", ] + }, + "data_split_parameters": { + "type": "object", + "properties": { + "test_size": {"type": "number"}, + "random_state": {"type": "integer"}, + }, + }, + "model_training_parameters": { + "type": "object", + "properties": { + "n_estimators": {"type": "integer", "default": 1000} + }, + }, + }, + "required": [ + "enabled", + "train_period_days", + "backtest_period_days", + "identifier", + "feature_parameters", + "data_split_parameters", + "model_training_parameters" + ] + }, }, } diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index b9b118c00..21cead77f 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -5,6 +5,7 @@ including ticker and orderbook data, live and historical candle (OHLCV) data Common Interface for bot and strategy to access data. """ import logging +from collections import deque from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple @@ -16,6 +17,7 @@ from freqtrade.data.history import load_pair_history from freqtrade.enums import CandleType, RunMode from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.exchange import Exchange, timeframe_to_seconds +from freqtrade.util import PeriodicCache logger = logging.getLogger(__name__) @@ -33,6 +35,10 @@ class DataProvider: self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} self.__slice_index: Optional[int] = None self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {} + self._msg_queue: deque = deque() + + self.__msg_cache = PeriodicCache( + maxsize=1000, ttl=timeframe_to_seconds(self._config.get('timeframe', '1h'))) def _set_dataframe_max_index(self, limit_index: int): """ @@ -265,3 +271,20 @@ class DataProvider: if self._exchange is None: raise OperationalException(NO_EXCHANGE_EXCEPTION) return self._exchange.fetch_l2_order_book(pair, maximum) + + def send_msg(self, message: str, *, always_send: bool = False) -> None: + """ + Send custom RPC Notifications from your bot. + Will not send any bot in modes other than Dry-run or Live. + :param message: Message to be sent. Must be below 4096. + :param always_send: If False, will send the message only once per candle, and surpress + identical messages. + Careful as this can end up spaming your chat. + Defaults to False + """ + if self.runmode not in (RunMode.DRY_RUN, RunMode.LIVE): + return + + if always_send or message not in self.__msg_cache: + self._msg_queue.append(message) + self.__msg_cache[message] = True diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index dadc9c7e6..135d97c79 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -7,9 +7,8 @@ import numpy as np import pandas as pd from freqtrade.configuration import TimeRange -from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, - ListPairsWithTimeframes, TradeList) -from freqtrade.enums import CandleType, TradingMode +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList +from freqtrade.enums import CandleType from .idatahandler import IDataHandler @@ -21,29 +20,6 @@ class HDF5DataHandler(IDataHandler): _columns = DEFAULT_DATAFRAME_COLUMNS - @classmethod - def ohlcv_get_available_data( - cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes: - """ - Returns a list of all pairs with ohlcv data available in this datadir - :param datadir: Directory to search for ohlcv files - :param trading_mode: trading-mode to be used - :return: List of Tuples of (pair, timeframe) - """ - if trading_mode == TradingMode.FUTURES: - datadir = datadir.joinpath('futures') - _tmp = [ - re.search( - cls._OHLCV_REGEX, p.name - ) for p in datadir.glob("*.h5") - ] - return [ - ( - cls.rebuild_pair_from_filename(match[1]), - cls.rebuild_timeframe_from_filename(match[2]), - CandleType.from_string(match[3]) - ) for match in _tmp if match and len(match.groups()) > 1] - @classmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]: """ diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index c972c841c..7a3fa4e0c 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -56,7 +56,7 @@ def load_pair_history(pair: str, fill_missing=fill_up_missing, drop_incomplete=drop_incomplete, startup_candles=startup_candles, - candle_type=candle_type + candle_type=candle_type, ) @@ -97,14 +97,15 @@ def load_data(datadir: Path, fill_up_missing=fill_up_missing, startup_candles=startup_candles, data_handler=data_handler, - candle_type=candle_type + candle_type=candle_type, ) if not hist.empty: result[pair] = hist else: if candle_type is CandleType.FUNDING_RATE and user_futures_funding_rate is not None: logger.warn(f"{pair} using user specified [{user_futures_funding_rate}]") - result[pair] = DataFrame(columns=["open", "close", "high", "low", "volume"]) + elif candle_type not in (CandleType.SPOT, CandleType.FUTURES): + result[pair] = DataFrame(columns=["date", "open", "close", "high", "low", "volume"]) if fail_without_data and not result: raise OperationalException("No data found. Terminating.") @@ -301,8 +302,8 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes if trading_mode == 'futures': # Predefined candletype (and timeframe) depending on exchange # Downloads what is necessary to backtest based on futures data. - tf_mark = exchange._ft_has['mark_ohlcv_timeframe'] - fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price']) + tf_mark = exchange.get_option('mark_ohlcv_timeframe') + fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price')) # All exchanges need FundingRate for futures trading. # The timeframe is aligned to the mark-price timeframe. for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type): @@ -329,13 +330,12 @@ def _download_trades_history(exchange: Exchange, try: until = None + since = 0 if timerange: if timerange.starttype == 'date': since = timerange.startts * 1000 if timerange.stoptype == 'date': until = timerange.stopts * 1000 - else: - since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 trades = data_handler.trades_load(pair) @@ -348,6 +348,9 @@ def _download_trades_history(exchange: Exchange, logger.info(f"Start earlier than available data. Redownloading trades for {pair}...") trades = [] + if not since: + since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 + from_id = trades[-1][1] if trades else None if trades and since < trades[-1][0]: # Reset since to the last available point diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 07dc7c763..846bcc607 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -9,7 +9,7 @@ from abc import ABC, abstractmethod from copy import deepcopy from datetime import datetime, timezone from pathlib import Path -from typing import List, Optional, Type +from typing import List, Optional, Tuple, Type from pandas import DataFrame @@ -39,15 +39,26 @@ class IDataHandler(ABC): raise NotImplementedError() @classmethod - @abstractmethod def ohlcv_get_available_data( cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes: """ Returns a list of all pairs with ohlcv data available in this datadir :param datadir: Directory to search for ohlcv files :param trading_mode: trading-mode to be used - :return: List of Tuples of (pair, timeframe) + :return: List of Tuples of (pair, timeframe, CandleType) """ + if trading_mode == TradingMode.FUTURES: + datadir = datadir.joinpath('futures') + _tmp = [ + re.search( + cls._OHLCV_REGEX, p.name + ) for p in datadir.glob(f"*.{cls._get_file_extension()}")] + return [ + ( + cls.rebuild_pair_from_filename(match[1]), + cls.rebuild_timeframe_from_filename(match[2]), + CandleType.from_string(match[3]) + ) for match in _tmp if match and len(match.groups()) > 1] @classmethod @abstractmethod @@ -73,6 +84,18 @@ class IDataHandler(ABC): :return: None """ + def ohlcv_data_min_max(self, pair: str, timeframe: str, + candle_type: CandleType) -> Tuple[datetime, datetime]: + """ + Returns the min and max timestamp for the given pair and timeframe. + :param pair: Pair to get min/max for + :param timeframe: Timeframe to get min/max for + :param candle_type: Any of the enum CandleType (must match trading mode!) + :return: (min, max) + """ + data = self._ohlcv_load(pair, timeframe, None, candle_type) + return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime() + @abstractmethod def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange], candle_type: CandleType diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 83ec183df..a62e5e381 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -8,9 +8,9 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes, TradeList +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, TradeList from freqtrade.data.converter import trades_dict_to_list -from freqtrade.enums import CandleType, TradingMode +from freqtrade.enums import CandleType from .idatahandler import IDataHandler @@ -23,28 +23,6 @@ class JsonDataHandler(IDataHandler): _use_zip = False _columns = DEFAULT_DATAFRAME_COLUMNS - @classmethod - def ohlcv_get_available_data( - cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes: - """ - Returns a list of all pairs with ohlcv data available in this datadir - :param datadir: Directory to search for ohlcv files - :param trading_mode: trading-mode to be used - :return: List of Tuples of (pair, timeframe) - """ - if trading_mode == 'futures': - datadir = datadir.joinpath('futures') - _tmp = [ - re.search( - cls._OHLCV_REGEX, p.name - ) for p in datadir.glob(f"*.{cls._get_file_extension()}")] - return [ - ( - cls.rebuild_pair_from_filename(match[1]), - cls.rebuild_timeframe_from_filename(match[2]), - CandleType.from_string(match[3]) - ) for match in _tmp if match and len(match.groups()) > 1] - @classmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]: """ diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 2fe41a17b..af20e1645 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -15,7 +15,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT from freqtrade.data.history import get_timerange, load_data, refresh_data from freqtrade.enums import CandleType, ExitType, RunMode from freqtrade.exceptions import OperationalException -from freqtrade.exchange.exchange import timeframe_to_seconds +from freqtrade.exchange import timeframe_to_seconds from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.strategy.interface import IStrategy diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py index e50ebc4a4..d2f5474fc 100644 --- a/freqtrade/enums/__init__.py +++ b/freqtrade/enums/__init__.py @@ -3,6 +3,7 @@ from freqtrade.enums.backteststate import BacktestState from freqtrade.enums.candletype import CandleType from freqtrade.enums.exitchecktuple import ExitCheckTuple from freqtrade.enums.exittype import ExitType +from freqtrade.enums.hyperoptstate import HyperoptState from freqtrade.enums.marginmode import MarginMode from freqtrade.enums.ordertypevalue import OrderTypeValues from freqtrade.enums.rpcmessagetype import RPCMessageType diff --git a/freqtrade/enums/exittype.py b/freqtrade/enums/exittype.py index b2c5b62ea..b025230ba 100644 --- a/freqtrade/enums/exittype.py +++ b/freqtrade/enums/exittype.py @@ -9,10 +9,12 @@ class ExitType(Enum): STOP_LOSS = "stop_loss" STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange" TRAILING_STOP_LOSS = "trailing_stop_loss" + LIQUIDATION = "liquidation" EXIT_SIGNAL = "exit_signal" FORCE_EXIT = "force_exit" EMERGENCY_EXIT = "emergency_exit" CUSTOM_EXIT = "custom_exit" + PARTIAL_EXIT = "partial_exit" NONE = "" def __str__(self): diff --git a/freqtrade/enums/hyperoptstate.py b/freqtrade/enums/hyperoptstate.py new file mode 100644 index 000000000..6716e123a --- /dev/null +++ b/freqtrade/enums/hyperoptstate.py @@ -0,0 +1,12 @@ +from enum import Enum + + +class HyperoptState(Enum): + """ Hyperopt states """ + STARTUP = 1 + DATALOAD = 2 + INDICATORS = 3 + OPTIMIZE = 4 + + def __str__(self): + return f"{self.name.lower()}" diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py index 584a011c2..415d8f18c 100644 --- a/freqtrade/enums/rpcmessagetype.py +++ b/freqtrade/enums/rpcmessagetype.py @@ -17,6 +17,8 @@ class RPCMessageType(Enum): PROTECTION_TRIGGER = 'protection_trigger' PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global' + STRATEGY_MSG = 'strategy_msg' + def __repr__(self): return self.value diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 2b9ed47ea..ff7ec7e04 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -9,12 +9,14 @@ from freqtrade.exchange.bitpanda import Bitpanda from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.bybit import Bybit from freqtrade.exchange.coinbasepro import Coinbasepro -from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges, +from freqtrade.exchange.exchange import (amount_to_contract_precision, amount_to_contracts, + amount_to_precision, available_exchanges, ccxt_exchanges, + contracts_to_amount, date_minus_candles, is_exchange_known_ccxt, is_exchange_officially_supported, - market_is_active, timeframe_to_minutes, timeframe_to_msecs, - timeframe_to_next_date, timeframe_to_prev_date, - timeframe_to_seconds, validate_exchange, - validate_exchanges) + market_is_active, price_to_precision, timeframe_to_minutes, + timeframe_to_msecs, timeframe_to_next_date, + timeframe_to_prev_date, timeframe_to_seconds, + validate_exchange, validate_exchanges) from freqtrade.exchange.ftx import Ftx from freqtrade.exchange.gateio import Gateio from freqtrade.exchange.hitbtc import Hitbtc diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 37a3c419d..a5e9fd37c 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -137,23 +137,27 @@ class Binance(Exchange): pair: str, open_rate: float, # Entry price of position is_short: bool, - position: float, # Absolute value of position size + amount: float, + stake_amount: float, wallet_balance: float, # Or margin balance mm_ex_1: float = 0.0, # (Binance) Cross only upnl_ex_1: float = 0.0, # (Binance) Cross only ) -> Optional[float]: """ + Important: Must be fetching data from cached values as this is used by backtesting! MARGIN: https://www.binance.com/en/support/faq/f6b010588e55413aa58b7d63ee0125ed PERPETUAL: https://www.binance.com/en/support/faq/b3c689c1f50a44cabb3a84e663b81d93 :param exchange_name: - :param open_rate: (EP1) Entry price of position + :param open_rate: Entry price of position :param is_short: True if the trade is a short, false otherwise - :param position: Absolute value of position size (in base currency) - :param wallet_balance: (WB) + :param amount: Absolute value of position size incl. leverage (in base currency) + :param stake_amount: Stake amount - Collateral in settle currency. + :param trading_mode: SPOT, MARGIN, FUTURES, etc. + :param margin_mode: Either ISOLATED or CROSS + :param wallet_balance: Amount of margin_mode in the wallet being used to trade Cross-Margin Mode: crossWalletBalance Isolated-Margin Mode: isolatedWalletBalance - :param maintenance_amt: # * Only required for Cross :param mm_ex_1: (TMM) @@ -165,12 +169,11 @@ class Binance(Exchange): """ side_1 = -1 if is_short else 1 - position = abs(position) cross_vars = upnl_ex_1 - mm_ex_1 if self.margin_mode == MarginMode.CROSS else 0.0 # mm_ratio: Binance's formula specifies maintenance margin rate which is mm_ratio * 100% # maintenance_amt: (CUM) Maintenance Amount of position - mm_ratio, maintenance_amt = self.get_maintenance_ratio_and_amt(pair, position) + mm_ratio, maintenance_amt = self.get_maintenance_ratio_and_amt(pair, stake_amount) if (maintenance_amt is None): raise OperationalException( @@ -182,9 +185,9 @@ class Binance(Exchange): return ( ( (wallet_balance + cross_vars + maintenance_amt) - - (side_1 * position * open_rate) + (side_1 * amount * open_rate) ) / ( - (position * mm_ratio) - (side_1 * position) + (amount * mm_ratio) - (side_1 * amount) ) ) else: diff --git a/freqtrade/exchange/binance_leverage_tiers.json b/freqtrade/exchange/binance_leverage_tiers.json index 1cf6ba079..eace16c05 100644 --- a/freqtrade/exchange/binance_leverage_tiers.json +++ b/freqtrade/exchange/binance_leverage_tiers.json @@ -81,6 +81,88 @@ } } ], + "1000SHIB/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "1000SHIB/USDT": [ { "tier": 1.0, @@ -1991,6 +2073,88 @@ } } ], + "AUCTION/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "AUDIO/USDT": [ { "tier": 1.0, @@ -5333,6 +5497,88 @@ } } ], + "CVX/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "DAR/USDT": [ { "tier": 1.0, @@ -7013,6 +7259,88 @@ } } ], + "ETC/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "ETC/USDT": [ { "tier": 1.0, @@ -7164,13 +7492,13 @@ "tier": 1.0, "currency": "BUSD", "minNotional": 0.0, - "maxNotional": 25000.0, + "maxNotional": 50000.0, "maintenanceMarginRate": 0.004, "maxLeverage": 50.0, "info": { "bracket": "1", "initialLeverage": "50", - "notionalCap": "25000", + "notionalCap": "50000", "notionalFloor": "0", "maintMarginRatio": "0.004", "cum": "0.0" @@ -7179,7 +7507,7 @@ { "tier": 2.0, "currency": "BUSD", - "minNotional": 25000.0, + "minNotional": 50000.0, "maxNotional": 100000.0, "maintenanceMarginRate": 0.005, "maxLeverage": 25.0, @@ -7187,111 +7515,111 @@ "bracket": "2", "initialLeverage": "25", "notionalCap": "100000", - "notionalFloor": "25000", + "notionalFloor": "50000", "maintMarginRatio": "0.005", - "cum": "25.0" + "cum": "50.0" } }, { "tier": 3.0, "currency": "BUSD", "minNotional": 100000.0, - "maxNotional": 500000.0, + "maxNotional": 1000000.0, "maintenanceMarginRate": 0.01, "maxLeverage": 20.0, "info": { "bracket": "3", "initialLeverage": "20", - "notionalCap": "500000", + "notionalCap": "1000000", "notionalFloor": "100000", "maintMarginRatio": "0.01", - "cum": "525.0" + "cum": "550.0" } }, { "tier": 4.0, "currency": "BUSD", - "minNotional": 500000.0, - "maxNotional": 1500000.0, + "minNotional": 1000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.025, "maxLeverage": 10.0, "info": { "bracket": "4", "initialLeverage": "10", - "notionalCap": "1500000", - "notionalFloor": "500000", + "notionalCap": "5000000", + "notionalFloor": "1000000", "maintMarginRatio": "0.025", - "cum": "8025.0" + "cum": "15550.0" } }, { "tier": 5.0, "currency": "BUSD", - "minNotional": 1500000.0, - "maxNotional": 4000000.0, + "minNotional": 5000000.0, + "maxNotional": 10000000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 6.0, "info": { "bracket": "5", "initialLeverage": "6", - "notionalCap": "4000000", - "notionalFloor": "1500000", + "notionalCap": "10000000", + "notionalFloor": "5000000", "maintMarginRatio": "0.05", - "cum": "45525.0" + "cum": "140550.0" } }, { "tier": 6.0, "currency": "BUSD", - "minNotional": 4000000.0, - "maxNotional": 10000000.0, + "minNotional": 10000000.0, + "maxNotional": 20000000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { "bracket": "6", "initialLeverage": "5", - "notionalCap": "10000000", - "notionalFloor": "4000000", + "notionalCap": "20000000", + "notionalFloor": "10000000", "maintMarginRatio": "0.1", - "cum": "245525.0" + "cum": "640550.0" } }, { "tier": 7.0, "currency": "BUSD", - "minNotional": 10000000.0, - "maxNotional": 20000000.0, + "minNotional": 20000000.0, + "maxNotional": 40000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 4.0, "info": { "bracket": "7", "initialLeverage": "4", - "notionalCap": "20000000", - "notionalFloor": "10000000", + "notionalCap": "40000000", + "notionalFloor": "20000000", "maintMarginRatio": "0.125", - "cum": "495525.0" + "cum": "1140550.0" } }, { "tier": 8.0, "currency": "BUSD", - "minNotional": 20000000.0, - "maxNotional": 40000000.0, + "minNotional": 40000000.0, + "maxNotional": 80000000.0, "maintenanceMarginRate": 0.15, "maxLeverage": 3.0, "info": { "bracket": "8", "initialLeverage": "3", - "notionalCap": "40000000", - "notionalFloor": "20000000", + "notionalCap": "80000000", + "notionalFloor": "40000000", "maintMarginRatio": "0.15", - "cum": "995525.0" + "cum": "2140550.0" } }, { "tier": 9.0, "currency": "BUSD", - "minNotional": 40000000.0, + "minNotional": 80000000.0, "maxNotional": 150000000.0, "maintenanceMarginRate": 0.25, "maxLeverage": 2.0, @@ -7299,25 +7627,25 @@ "bracket": "9", "initialLeverage": "2", "notionalCap": "150000000", - "notionalFloor": "40000000", + "notionalFloor": "80000000", "maintMarginRatio": "0.25", - "cum": "4995525.0" + "cum": "1.014055E7" } }, { "tier": 10.0, "currency": "BUSD", "minNotional": 150000000.0, - "maxNotional": 500000000.0, + "maxNotional": 300000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "10", "initialLeverage": "1", - "notionalCap": "500000000", + "notionalCap": "300000000", "notionalFloor": "150000000", "maintMarginRatio": "0.5", - "cum": "4.2495525E7" + "cum": "4.764055E7" } } ], @@ -7326,13 +7654,13 @@ "tier": 1.0, "currency": "USDT", "minNotional": 0.0, - "maxNotional": 10000.0, + "maxNotional": 100000.0, "maintenanceMarginRate": 0.005, "maxLeverage": 100.0, "info": { "bracket": "1", "initialLeverage": "100", - "notionalCap": "10000", + "notionalCap": "100000", "notionalFloor": "0", "maintMarginRatio": "0.005", "cum": "0.0" @@ -7341,119 +7669,119 @@ { "tier": 2.0, "currency": "USDT", - "minNotional": 10000.0, - "maxNotional": 100000.0, + "minNotional": 100000.0, + "maxNotional": 250000.0, "maintenanceMarginRate": 0.0065, "maxLeverage": 75.0, "info": { "bracket": "2", "initialLeverage": "75", - "notionalCap": "100000", - "notionalFloor": "10000", + "notionalCap": "250000", + "notionalFloor": "100000", "maintMarginRatio": "0.0065", - "cum": "15.0" + "cum": "150.0" } }, { "tier": 3.0, "currency": "USDT", - "minNotional": 100000.0, - "maxNotional": 500000.0, + "minNotional": 250000.0, + "maxNotional": 1000000.0, "maintenanceMarginRate": 0.01, "maxLeverage": 50.0, "info": { "bracket": "3", "initialLeverage": "50", - "notionalCap": "500000", - "notionalFloor": "100000", + "notionalCap": "1000000", + "notionalFloor": "250000", "maintMarginRatio": "0.01", - "cum": "365.0" + "cum": "1025.0" } }, { "tier": 4.0, "currency": "USDT", - "minNotional": 500000.0, - "maxNotional": 1500000.0, + "minNotional": 1000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.02, "maxLeverage": 25.0, "info": { "bracket": "4", "initialLeverage": "25", - "notionalCap": "1500000", - "notionalFloor": "500000", + "notionalCap": "5000000", + "notionalFloor": "1000000", "maintMarginRatio": "0.02", - "cum": "5365.0" + "cum": "11025.0" } }, { "tier": 5.0, "currency": "USDT", - "minNotional": 1500000.0, - "maxNotional": 4000000.0, + "minNotional": 5000000.0, + "maxNotional": 10000000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 10.0, "info": { "bracket": "5", "initialLeverage": "10", - "notionalCap": "4000000", - "notionalFloor": "1500000", + "notionalCap": "10000000", + "notionalFloor": "5000000", "maintMarginRatio": "0.05", - "cum": "50365.0" + "cum": "161025.0" } }, { "tier": 6.0, "currency": "USDT", - "minNotional": 4000000.0, - "maxNotional": 10000000.0, + "minNotional": 10000000.0, + "maxNotional": 20000000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { "bracket": "6", "initialLeverage": "5", - "notionalCap": "10000000", - "notionalFloor": "4000000", + "notionalCap": "20000000", + "notionalFloor": "10000000", "maintMarginRatio": "0.1", - "cum": "250365.0" + "cum": "661025.0" } }, { "tier": 7.0, "currency": "USDT", - "minNotional": 10000000.0, - "maxNotional": 20000000.0, + "minNotional": 20000000.0, + "maxNotional": 40000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 4.0, "info": { "bracket": "7", "initialLeverage": "4", - "notionalCap": "20000000", - "notionalFloor": "10000000", + "notionalCap": "40000000", + "notionalFloor": "20000000", "maintMarginRatio": "0.125", - "cum": "500365.0" + "cum": "1161025.0" } }, { "tier": 8.0, "currency": "USDT", - "minNotional": 20000000.0, - "maxNotional": 40000000.0, + "minNotional": 40000000.0, + "maxNotional": 80000000.0, "maintenanceMarginRate": 0.15, "maxLeverage": 3.0, "info": { "bracket": "8", "initialLeverage": "3", - "notionalCap": "40000000", - "notionalFloor": "20000000", + "notionalCap": "80000000", + "notionalFloor": "40000000", "maintMarginRatio": "0.15", - "cum": "1000365.0" + "cum": "2161025.0" } }, { "tier": 9.0, "currency": "USDT", - "minNotional": 40000000.0, + "minNotional": 80000000.0, "maxNotional": 150000000.0, "maintenanceMarginRate": 0.25, "maxLeverage": 2.0, @@ -7461,25 +7789,25 @@ "bracket": "9", "initialLeverage": "2", "notionalCap": "150000000", - "notionalFloor": "40000000", + "notionalFloor": "80000000", "maintMarginRatio": "0.25", - "cum": "5000365.0" + "cum": "1.0161025E7" } }, { "tier": 10.0, "currency": "USDT", "minNotional": 150000000.0, - "maxNotional": 500000000.0, + "maxNotional": 300000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "10", "initialLeverage": "1", - "notionalCap": "500000000", + "notionalCap": "300000000", "notionalFloor": "150000000", "maintMarginRatio": "0.5", - "cum": "4.2500365E7" + "cum": "4.7661025E7" } } ], @@ -7597,6 +7925,88 @@ } } ], + "FIL/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "FIL/USDT": [ { "tier": 1.0, @@ -9737,6 +10147,104 @@ } } ], + "INJ/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.01, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.01", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "75.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "700.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11950.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386950.0" + } + } + ], "IOST/USDT": [ { "tier": 1.0, @@ -10521,6 +11029,170 @@ } } ], + "LDO/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], + "LEVER/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "LINA/USDT": [ { "tier": 1.0, @@ -11663,6 +12335,88 @@ } } ], + "MATIC/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "MATIC/USDT": [ { "tier": 1.0, @@ -16055,6 +16809,88 @@ } } ], + "UNI/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "UNI/USDT": [ { "tier": 1.0, diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 79bc769e6..4386f47f6 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -16,7 +16,8 @@ import arrow import ccxt import ccxt.async_support as ccxt_async from cachetools import TTLCache -from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, Precise, decimal_to_precision +from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision +from dateutil import parser from pandas import DataFrame from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell, @@ -30,8 +31,10 @@ from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGE EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, SUPPORTED_EXCHANGES, remove_credentials, retrier, retrier_async) -from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 +from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json, + safe_value_fallback2) from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist +from freqtrade.util import FtPrecise CcxtModuleType = Any @@ -51,8 +54,8 @@ class Exchange: # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) _params: Dict = {} - # Additional headers - added to the ccxt object - _headers: Dict = {} + # Additional parameters - added to the ccxt object + _ccxt_params: Dict = {} # Dict to specify which options each exchange implements # This defines defaults, which can be selectively overridden by subclasses using _ft_has @@ -115,6 +118,7 @@ class Exchange: self._last_markets_refresh: int = 0 # Cache for 10 minutes ... + self._cache_lock = Lock() self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=2, ttl=60 * 10) # Cache values for 1800 to avoid frequent polling of the exchange for prices # Caching only applies to RPC methods, so prices for open trades are still @@ -238,9 +242,9 @@ class Exchange: } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) - if self._headers: - # Inject static headers after the above output to not confuse users. - ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs) + if self._ccxt_params: + # Inject static options after the above output to not confuse users. + ccxt_kwargs = deep_merge_dicts(self._ccxt_params, ccxt_kwargs) if ccxt_kwargs: ex_config.update(ccxt_kwargs) try: @@ -404,7 +408,7 @@ class Exchange: else: return DataFrame() - def _get_contract_size(self, pair: str) -> float: + def get_contract_size(self, pair: str) -> float: if self.trading_mode == TradingMode.FUTURES: market = self.markets[pair] contract_size: float = 1.0 @@ -417,7 +421,7 @@ class Exchange: def _trades_contracts_to_amount(self, trades: List) -> List: if len(trades) > 0 and 'symbol' in trades[0]: - contract_size = self._get_contract_size(trades[0]['symbol']) + contract_size = self.get_contract_size(trades[0]['symbol']) if contract_size != 1: for trade in trades: trade['amount'] = trade['amount'] * contract_size @@ -425,7 +429,7 @@ class Exchange: def _order_contracts_to_amount(self, order: Dict) -> Dict: if 'symbol' in order and order['symbol'] is not None: - contract_size = self._get_contract_size(order['symbol']) + contract_size = self.get_contract_size(order['symbol']) if contract_size != 1: for prop in self._ft_has.get('order_props_in_contracts', []): if prop in order and order[prop] is not None: @@ -434,19 +438,13 @@ class Exchange: def _amount_to_contracts(self, pair: str, amount: float) -> float: - contract_size = self._get_contract_size(pair) - if contract_size and contract_size != 1: - return amount / contract_size - else: - return amount + contract_size = self.get_contract_size(pair) + return amount_to_contracts(amount, contract_size) def _contracts_to_amount(self, pair: str, num_contracts: float) -> float: - contract_size = self._get_contract_size(pair) - if contract_size and contract_size != 1: - return num_contracts * contract_size - else: - return num_contracts + contract_size = self.get_contract_size(pair) + return contracts_to_amount(num_contracts, contract_size) def set_sandbox(self, api: ccxt.Exchange, exchange_config: dict, name: str) -> None: if exchange_config.get('sandbox'): @@ -670,6 +668,12 @@ class Exchange: f"Freqtrade does not support {mm_value} {trading_mode.value} on {self.name}" ) + def get_option(self, param: str, default: Any = None) -> Any: + """ + Get parameter value from _ft_has + """ + return self._ft_has.get(param, default) + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. @@ -679,45 +683,35 @@ class Exchange: """ return endpoint in self._api.has and self._api.has[endpoint] + def get_precision_amount(self, pair: str) -> Optional[float]: + """ + Returns the amount precision of the exchange. + :param pair: Pair to get precision for + :return: precision for amount or None. Must be used in combination with precisionMode + """ + return self.markets.get(pair, {}).get('precision', {}).get('amount', None) + + def get_precision_price(self, pair: str) -> Optional[float]: + """ + Returns the price precision of the exchange. + :param pair: Pair to get precision for + :return: precision for price or None. Must be used in combination with precisionMode + """ + return self.markets.get(pair, {}).get('precision', {}).get('price', None) + def amount_to_precision(self, pair: str, amount: float) -> float: """ Returns the amount to buy or sell to a precision the Exchange accepts - Re-implementation of ccxt internal methods - ensuring we can test the result is correct - based on our definitions. - """ - if self.markets[pair]['precision']['amount'] is not None: - amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE, - precision=self.markets[pair]['precision']['amount'], - counting_mode=self.precisionMode, - )) - return amount + """ + return amount_to_precision(amount, self.get_precision_amount(pair), self.precisionMode) def price_to_precision(self, pair: str, price: float) -> float: """ Returns the price rounded up to the precision the Exchange accepts. - Partial Re-implementation of ccxt internal method decimal_to_precision(), - which does not support rounding up - TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and - align with amount_to_precision(). Rounds up """ - if self.markets[pair]['precision']['price']: - # price = float(decimal_to_precision(price, rounding_mode=ROUND, - # precision=self.markets[pair]['precision']['price'], - # counting_mode=self.precisionMode, - # )) - if self.precisionMode == TICK_SIZE: - precision = Precise(str(self.markets[pair]['precision']['price'])) - price_str = Precise(str(price)) - missing = price_str % precision - if not missing == Precise("0"): - price = round(float(str(price_str - missing + precision)), 14) - else: - symbol_prec = self.markets[pair]['precision']['price'] - big_price = price * pow(10, symbol_prec) - price = ceil(big_price) / pow(10, symbol_prec) - return price + return price_to_precision(price, self.get_precision_price(pair), self.precisionMode) def price_get_one_pip(self, pair: str, price: float) -> float: """ @@ -849,6 +843,7 @@ class Exchange: dry_order.update({ 'average': average, 'filled': _amount, + 'remaining': 0.0, 'cost': (dry_order['amount'] * average) / leverage }) # market orders will always incurr taker fees @@ -1017,7 +1012,8 @@ class Exchange: time_in_force: str = 'gtc', ) -> Dict: if self._config['dry_run']: - dry_order = self.create_dry_run_order(pair, ordertype, side, amount, rate, leverage) + dry_order = self.create_dry_run_order( + pair, ordertype, side, amount, self.price_to_precision(pair, rate), leverage) return dry_order params = self._get_params(side, ordertype, leverage, reduceOnly, time_in_force) @@ -1332,11 +1328,19 @@ class Exchange: raise OperationalException(e) from e @retrier - def fetch_positions(self) -> List[Dict]: + def fetch_positions(self, pair: str = None) -> List[Dict]: + """ + Fetch positions from the exchange. + If no pair is given, all positions are returned. + :param pair: Pair for the query + """ if self._config['dry_run'] or self.trading_mode != TradingMode.FUTURES: return [] try: - positions: List[Dict] = self._api.fetch_positions() + symbols = [] + if pair: + symbols.append(pair) + positions: List[Dict] = self._api.fetch_positions(symbols) self._log_exchange_response('fetch_positions', positions) return positions except ccxt.DDoSProtection as e: @@ -1377,12 +1381,14 @@ class Exchange: if not self.exchange_has('fetchBidsAsks'): return {} if cached: - tickers = self._fetch_tickers_cache.get('fetch_bids_asks') + with self._cache_lock: + tickers = self._fetch_tickers_cache.get('fetch_bids_asks') if tickers: return tickers try: tickers = self._api.fetch_bids_asks(symbols) - self._fetch_tickers_cache['fetch_bids_asks'] = tickers + with self._cache_lock: + self._fetch_tickers_cache['fetch_bids_asks'] = tickers return tickers except ccxt.NotSupported as e: raise OperationalException( @@ -1403,12 +1409,14 @@ class Exchange: :return: fetch_tickers result """ if cached: - tickers = self._fetch_tickers_cache.get('fetch_tickers') + with self._cache_lock: + tickers = self._fetch_tickers_cache.get('fetch_tickers') if tickers: return tickers try: tickers = self._api.fetch_tickers(symbols) - self._fetch_tickers_cache['fetch_tickers'] = tickers + with self._cache_lock: + self._fetch_tickers_cache['fetch_tickers'] = tickers return tickers except ccxt.NotSupported as e: raise OperationalException( @@ -1499,7 +1507,8 @@ class Exchange: return price_side def get_rate(self, pair: str, refresh: bool, - side: EntryExit, is_short: bool) -> float: + side: EntryExit, is_short: bool, + order_book: Optional[dict] = None, ticker: Optional[dict] = None) -> float: """ Calculates bid/ask target bid rate - between current ask price and last price @@ -1516,7 +1525,8 @@ class Exchange: cache_rate: TTLCache = self._entry_rate_cache if side == "entry" else self._exit_rate_cache if not refresh: - rate = cache_rate.get(pair) + with self._cache_lock: + rate = cache_rate.get(pair) # Check if cache has been invalidated if rate: logger.debug(f"Using cached {side} rate for {pair}.") @@ -1531,22 +1541,24 @@ class Exchange: if conf_strategy.get('use_order_book', False): order_book_top = conf_strategy.get('order_book_top', 1) - order_book = self.fetch_l2_order_book(pair, order_book_top) + if order_book is None: + order_book = self.fetch_l2_order_book(pair, order_book_top) logger.debug('order_book %s', order_book) # top 1 = index 0 try: rate = order_book[f"{price_side}s"][order_book_top - 1][0] except (IndexError, KeyError) as e: logger.warning( - f"{name} Price at location {order_book_top} from orderbook could not be " - f"determined. Orderbook: {order_book}" + f"{pair} - {name} Price at location {order_book_top} from orderbook " + f"could not be determined. Orderbook: {order_book}" ) raise PricingError from e - logger.debug(f"{name} price from orderbook {price_side_word}" + logger.debug(f"{pair} - {name} price from orderbook {price_side_word}" f"side - top {order_book_top} order book {side} rate {rate:.8f}") else: logger.debug(f"Using Last {price_side_word} / Last Price") - ticker = self.fetch_ticker(pair) + if ticker is None: + ticker = self.fetch_ticker(pair) ticker_rate = ticker[price_side] if ticker['last'] and ticker_rate: if side == 'entry' and ticker_rate > ticker['last']: @@ -1559,10 +1571,39 @@ class Exchange: if rate is None: raise PricingError(f"{name}-Rate for {pair} was empty.") - cache_rate[pair] = rate + with self._cache_lock: + cache_rate[pair] = rate return rate + def get_rates(self, pair: str, refresh: bool, is_short: bool) -> Tuple[float, float]: + entry_rate = None + exit_rate = None + if not refresh: + with self._cache_lock: + entry_rate = self._entry_rate_cache.get(pair) + exit_rate = self._exit_rate_cache.get(pair) + if entry_rate: + logger.debug(f"Using cached buy rate for {pair}.") + if exit_rate: + logger.debug(f"Using cached sell rate for {pair}.") + + entry_pricing = self._config.get('entry_pricing', {}) + exit_pricing = self._config.get('exit_pricing', {}) + order_book = ticker = None + if not entry_rate and entry_pricing.get('use_order_book', False): + order_book_top = max(entry_pricing.get('order_book_top', 1), + exit_pricing.get('order_book_top', 1)) + order_book = self.fetch_l2_order_book(pair, order_book_top) + entry_rate = self.get_rate(pair, refresh, 'entry', is_short, order_book=order_book) + elif not entry_rate: + ticker = self.fetch_ticker(pair) + entry_rate = self.get_rate(pair, refresh, 'entry', is_short, ticker=ticker) + if not exit_rate: + exit_rate = self.get_rate(pair, refresh, 'exit', + is_short, order_book=order_book, ticker=ticker) + return entry_rate, exit_rate + # Fee handling @retrier @@ -1981,7 +2022,7 @@ class Exchange: else: logger.debug( "Fetching trades for pair %s, since %s %s...", - pair, since, + pair, since, '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else '' ) trades = await self._api_async.fetch_trades(pair, since=since, limit=1000) @@ -2168,6 +2209,7 @@ class Exchange: @retrier_async async def get_market_leverage_tiers(self, symbol: str) -> Tuple[str, List[Dict]]: + """ Leverage tiers per symbol """ try: tier = await self._api_async.fetch_market_leverage_tiers(symbol) return symbol, tier @@ -2199,20 +2241,34 @@ class Exchange: tiers: Dict[str, List[Dict]] = {} - # Be verbose here, as this delays startup by ~1 minute. - logger.info( - f"Initializing leverage_tiers for {len(symbols)} markets. " - "This will take about a minute.") + tiers_cached = self.load_cached_leverage_tiers(self._config['stake_currency']) + if tiers_cached: + tiers = tiers_cached - coros = [self.get_market_leverage_tiers(symbol) for symbol in sorted(symbols)] + coros = [ + self.get_market_leverage_tiers(symbol) + for symbol in sorted(symbols) if symbol not in tiers] + + # Be verbose here, as this delays startup by ~1 minute. + if coros: + logger.info( + f"Initializing leverage_tiers for {len(symbols)} markets. " + "This will take about a minute.") + else: + logger.info("Using cached leverage_tiers.") + + async def gather_results(): + return await asyncio.gather(*input_coro, return_exceptions=True) for input_coro in chunks(coros, 100): - results = self.loop.run_until_complete( - asyncio.gather(*input_coro, return_exceptions=True)) + with self._loop_lock: + results = self.loop.run_until_complete(gather_results()) + for symbol, res in results: tiers[symbol] = res - + if len(coros) > 0: + self.cache_leverage_tiers(tiers, self._config['stake_currency']) logger.info(f"Done initializing {len(symbols)} markets.") return tiers @@ -2221,6 +2277,30 @@ class Exchange: else: return {} + def cache_leverage_tiers(self, tiers: Dict[str, List[Dict]], stake_currency: str) -> None: + + filename = self._config['datadir'] / "futures" / f"leverage_tiers_{stake_currency}.json" + if not filename.parent.is_dir(): + filename.parent.mkdir(parents=True) + data = { + "updated": datetime.now(timezone.utc), + "data": tiers, + } + file_dump_json(filename, data) + + def load_cached_leverage_tiers(self, stake_currency: str) -> Optional[Dict[str, List[Dict]]]: + filename = self._config['datadir'] / "futures" / f"leverage_tiers_{stake_currency}.json" + if filename.is_file(): + tiers = file_load_json(filename) + updated = tiers.get('updated') + if updated: + updated_dt = parser.parse(updated) + if updated_dt < datetime.now(timezone.utc) - timedelta(days=1): + logger.info("Cached leverage tiers are outdated. Will update.") + return None + return tiers['data'] + return None + def fill_leverage_tiers(self) -> None: """ Assigns property _leverage_tiers to a dictionary of information about the leverage @@ -2236,10 +2316,10 @@ class Exchange: def parse_leverage_tier(self, tier) -> Dict: info = tier.get('info', {}) return { - 'min': tier['minNotional'], - 'max': tier['maxNotional'], - 'mmr': tier['maintenanceMarginRate'], - 'lev': tier['maxLeverage'], + 'minNotional': tier['minNotional'], + 'maxNotional': tier['maxNotional'], + 'maintenanceMarginRate': tier['maintenanceMarginRate'], + 'maxLeverage': tier['maxLeverage'], 'maintAmt': float(info['cum']) if 'cum' in info else None, } @@ -2268,18 +2348,18 @@ class Exchange: pair_tiers = self._leverage_tiers[pair] if stake_amount == 0: - return self._leverage_tiers[pair][0]['lev'] # Max lev for lowest amount + return self._leverage_tiers[pair][0]['maxLeverage'] # Max lev for lowest amount for tier_index in range(len(pair_tiers)): tier = pair_tiers[tier_index] - lev = tier['lev'] + lev = tier['maxLeverage'] if tier_index < len(pair_tiers) - 1: next_tier = pair_tiers[tier_index + 1] - next_floor = next_tier['min'] / next_tier['lev'] + next_floor = next_tier['minNotional'] / next_tier['maxLeverage'] if next_floor > stake_amount: # Next tier min too high for stake amount - return min((tier['max'] / stake_amount), lev) + return min((tier['maxNotional'] / stake_amount), lev) # # With the two leverage tiers below, # - a stake amount of 150 would mean a max leverage of (10000 / 150) = 66.66 @@ -2300,10 +2380,11 @@ class Exchange: # else: # if on the last tier - if stake_amount > tier['max']: # If stake is > than max tradeable amount + if stake_amount > tier['maxNotional']: + # If stake is > than max tradeable amount raise InvalidOrderException(f'Amount {stake_amount} too high for {pair}') else: - return tier['lev'] + return tier['maxLeverage'] raise OperationalException( 'Looped through all tiers without finding a max leverage. Should never be reached' @@ -2334,7 +2415,8 @@ class Exchange: return try: - self._api.set_leverage(symbol=pair, leverage=leverage) + res = self._api.set_leverage(symbol=pair, leverage=leverage) + self._log_exchange_response('set_leverage', res) except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: @@ -2355,6 +2437,7 @@ class Exchange: pair: str, open_rate: float, amount: float, # quote currency, includes leverage + stake_amount: float, leverage: float, is_short: bool ) -> Optional[float]: @@ -2362,23 +2445,22 @@ class Exchange: if self.trading_mode in TradingMode.SPOT: return None elif ( - self.margin_mode == MarginMode.ISOLATED and self.trading_mode == TradingMode.FUTURES ): - wallet_balance = (amount * open_rate) / leverage isolated_liq = self.get_or_calculate_liquidation_price( pair=pair, open_rate=open_rate, is_short=is_short, - position=amount, - wallet_balance=wallet_balance, + amount=amount, + stake_amount=stake_amount, + wallet_balance=stake_amount, # In isolated mode, stake-amount = wallet size mm_ex_1=0.0, upnl_ex_1=0.0, ) return isolated_liq else: raise OperationalException( - "Freqtrade only supports isolated futures for leverage trading") + "Freqtrade currently only supports futures for leverage trading.") def funding_fee_cutoff(self, open_date: datetime): """ @@ -2398,7 +2480,8 @@ class Exchange: return try: - self._api.set_margin_mode(margin_mode.value, pair, params) + res = self._api.set_margin_mode(margin_mode.value, pair, params) + self._log_exchange_response('set_margin_mode', res) except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: @@ -2539,25 +2622,24 @@ class Exchange: else: return 0.0 - @retrier def get_or_calculate_liquidation_price( self, pair: str, # Dry-run open_rate: float, # Entry price of position is_short: bool, - position: float, # Absolute value of position size + amount: float, # Absolute value of position size + stake_amount: float, wallet_balance: float, # Or margin balance mm_ex_1: float = 0.0, # (Binance) Cross only upnl_ex_1: float = 0.0, # (Binance) Cross only ) -> Optional[float]: """ Set's the margin mode on the exchange to cross or isolated for a specific pair - :param pair: base/quote currency pair (e.g. "ADA/USDT") """ if self.trading_mode == TradingMode.SPOT: return None - elif (self.trading_mode != TradingMode.FUTURES and self.margin_mode != MarginMode.ISOLATED): + elif (self.trading_mode != TradingMode.FUTURES): raise OperationalException( f"{self.name} does not support {self.margin_mode.value} {self.trading_mode.value}") @@ -2567,26 +2649,19 @@ class Exchange: pair=pair, open_rate=open_rate, is_short=is_short, - position=position, + amount=amount, + stake_amount=stake_amount, wallet_balance=wallet_balance, mm_ex_1=mm_ex_1, upnl_ex_1=upnl_ex_1 ) else: - try: - positions = self._api.fetch_positions([pair]) - if len(positions) > 0: - pos = positions[0] - isolated_liq = pos['liquidationPrice'] - else: - return None - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not set margin mode due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e + positions = self.fetch_positions(pair) + if len(positions) > 0: + pos = positions[0] + isolated_liq = pos['liquidationPrice'] + else: + return None if isolated_liq: buffer_amount = abs(open_rate - isolated_liq) * self.liquidation_buffer @@ -2604,22 +2679,24 @@ class Exchange: pair: str, open_rate: float, # Entry price of position is_short: bool, - position: float, # Absolute value of position size + amount: float, + stake_amount: float, wallet_balance: float, # Or margin balance mm_ex_1: float = 0.0, # (Binance) Cross only upnl_ex_1: float = 0.0, # (Binance) Cross only ) -> Optional[float]: """ + Important: Must be fetching data from cached values as this is used by backtesting! PERPETUAL: gateio: https://www.gate.io/help/futures/perpetual/22160/calculation-of-liquidation-price okex: https://www.okex.com/support/hc/en-us/articles/ 360053909592-VI-Introduction-to-the-isolated-mode-of-Single-Multi-currency-Portfolio-margin - Important: Must be fetching data from cached values as this is used by backtesting! :param exchange_name: :param open_rate: Entry price of position :param is_short: True if the trade is a short, false otherwise - :param position: Absolute value of position size incl. leverage (in base currency) + :param amount: Absolute value of position size incl. leverage (in base currency) + :param stake_amount: Stake amount - Collateral in settle currency. :param trading_mode: SPOT, MARGIN, FUTURES, etc. :param margin_mode: Either ISOLATED or CROSS :param wallet_balance: Amount of margin_mode in the wallet being used to trade @@ -2633,7 +2710,7 @@ class Exchange: market = self.markets[pair] taker_fee_rate = market['taker'] - mm_ratio, _ = self.get_maintenance_ratio_and_amt(pair, position) + mm_ratio, _ = self.get_maintenance_ratio_and_amt(pair, stake_amount) if self.trading_mode == TradingMode.FUTURES and self.margin_mode == MarginMode.ISOLATED: @@ -2641,7 +2718,7 @@ class Exchange: raise OperationalException( "Freqtrade does not yet support inverse contracts") - value = wallet_balance / position + value = wallet_balance / amount mm_ratio_taker = (mm_ratio + taker_fee_rate) if is_short: @@ -2677,8 +2754,8 @@ class Exchange: pair_tiers = self._leverage_tiers[pair] for tier in reversed(pair_tiers): - if nominal_value >= tier['min']: - return (tier['mmr'], tier['maintAmt']) + if nominal_value >= tier['minNotional']: + return (tier['maintenanceMarginRate'], tier['maintAmt']) raise OperationalException("nominal value can not be lower than 0") # The lowest notional_floor for any pair in fetch_leverage_tiers is always 0 because it @@ -2818,3 +2895,111 @@ def market_is_active(market: Dict) -> bool: # See https://github.com/ccxt/ccxt/issues/4874, # https://github.com/ccxt/ccxt/issues/4075#issuecomment-434760520 return market.get('active', True) is not False + + +def amount_to_contracts(amount: float, contract_size: Optional[float]) -> float: + """ + Convert amount to contracts. + :param amount: amount to convert + :param contract_size: contract size - taken from exchange.get_contract_size(pair) + :return: num-contracts + """ + if contract_size and contract_size != 1: + return amount / contract_size + else: + return amount + + +def contracts_to_amount(num_contracts: float, contract_size: Optional[float]) -> float: + """ + Takes num-contracts and converts it to contract size + :param num_contracts: number of contracts + :param contract_size: contract size - taken from exchange.get_contract_size(pair) + :return: Amount + """ + + if contract_size and contract_size != 1: + return num_contracts * contract_size + else: + return num_contracts + + +def amount_to_precision(amount: float, amount_precision: Optional[float], + precisionMode: Optional[int]) -> float: + """ + Returns the amount to buy or sell to a precision the Exchange accepts + Re-implementation of ccxt internal methods - ensuring we can test the result is correct + based on our definitions. + :param amount: amount to truncate + :param amount_precision: amount precision to use. + should be retrieved from markets[pair]['precision']['amount'] + :param precisionMode: precision mode to use. Should be used from precisionMode + one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE + :return: truncated amount + """ + if amount_precision is not None and precisionMode is not None: + precision = int(amount_precision) if precisionMode != TICK_SIZE else amount_precision + # precision must be an int for non-ticksize inputs. + amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE, + precision=precision, + counting_mode=precisionMode, + )) + + return amount + + +def amount_to_contract_precision( + amount, amount_precision: Optional[float], precisionMode: Optional[int], + contract_size: Optional[float]) -> float: + """ + Returns the amount to buy or sell to a precision the Exchange accepts + including calculation to and from contracts. + Re-implementation of ccxt internal methods - ensuring we can test the result is correct + based on our definitions. + :param amount: amount to truncate + :param amount_precision: amount precision to use. + should be retrieved from markets[pair]['precision']['amount'] + :param precisionMode: precision mode to use. Should be used from precisionMode + one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE + :param contract_size: contract size - taken from exchange.get_contract_size(pair) + :return: truncated amount + """ + if amount_precision is not None and precisionMode is not None: + contracts = amount_to_contracts(amount, contract_size) + amount_p = amount_to_precision(contracts, amount_precision, precisionMode) + return contracts_to_amount(amount_p, contract_size) + return amount + + +def price_to_precision(price: float, price_precision: Optional[float], + precisionMode: Optional[int]) -> float: + """ + Returns the price rounded up to the precision the Exchange accepts. + Partial Re-implementation of ccxt internal method decimal_to_precision(), + which does not support rounding up + TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and + align with amount_to_precision(). + !!! Rounds up + :param price: price to convert + :param price_precision: price precision to use. Used from markets[pair]['precision']['price'] + :param precisionMode: precision mode to use. Should be used from precisionMode + one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE + :return: price rounded up to the precision the Exchange accepts + + """ + if price_precision is not None and precisionMode is not None: + # price = float(decimal_to_precision(price, rounding_mode=ROUND, + # precision=price_precision, + # counting_mode=self.precisionMode, + # )) + if precisionMode == TICK_SIZE: + precision = FtPrecise(price_precision) + price_str = FtPrecise(price) + missing = price_str % precision + if not missing == FtPrecise("0"): + price = round(float(str(price_str - missing + precision)), 14) + else: + symbol_prec = price_precision + big_price = price * pow(10, symbol_prec) + price = ceil(big_price) / pow(10, symbol_prec) + return price diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index 9ee6894f1..b3c219542 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -1,6 +1,6 @@ """ FTX exchange subclass """ import logging -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Optional, Tuple import ccxt @@ -116,9 +116,17 @@ class Ftx(Exchange): if len(order) == 1: if order[0].get('status') == 'closed': # Trigger order was triggered ... - real_order_id = order[0].get('info', {}).get('orderId') + real_order_id: Optional[str] = order[0].get('info', {}).get('orderId') # OrderId may be None for stoploss-market orders - # But contains "average" in these cases. + # So we need to get it through the endpoint + # /conditional_orders/{conditional_order_id}/triggers + if not real_order_id: + res = self._api.privateGetConditionalOrdersConditionalOrderIdTriggers( + params={'conditional_order_id': order_id}) + self._log_exchange_response('fetch_stoploss_order2', res) + real_order_id = res['result'][0]['orderId'] if res.get( + 'result', []) else None + if real_order_id: order1 = self._api.fetch_order(real_order_id, pair) self._log_exchange_response('fetch_stoploss_order1', order1) diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index 6df3425d2..426a4b64d 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -25,7 +25,6 @@ class Gateio(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, - "ohlcv_volume_currency": "quote", "time_in_force_parameter": "timeInForce", "order_time_in_force": ['gtc', 'ioc'], "stoploss_order_types": {"limit": "limit"}, diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index afd7a672f..f039f2b3f 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -7,9 +7,8 @@ from freqtrade.constants import BuySell from freqtrade.enums import MarginMode, TradingMode from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError -from freqtrade.exchange import Exchange +from freqtrade.exchange import Exchange, date_minus_candles from freqtrade.exchange.common import retrier -from freqtrade.exchange.exchange import date_minus_candles logger = logging.getLogger(__name__) @@ -40,6 +39,8 @@ class Okx(Exchange): net_only = True + _ccxt_params: Dict = {'options': {'brokerId': 'ffb5405ad327SUDE'}} + def ohlcv_candle_limit( self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: """ @@ -145,4 +146,4 @@ class Okx(Exchange): return float('inf') pair_tiers = self._leverage_tiers[pair] - return pair_tiers[-1]['max'] / leverage + return pair_tiers[-1]['maxNotional'] / leverage diff --git a/freqtrade/freqai/__init__.py b/freqtrade/freqai/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py new file mode 100644 index 000000000..b6a1a15d7 --- /dev/null +++ b/freqtrade/freqai/data_drawer.py @@ -0,0 +1,608 @@ +import collections +import json +import logging +import re +import shutil +import threading +from pathlib import Path +from typing import Any, Dict, Tuple, TypedDict + +import numpy as np +import pandas as pd +import rapidjson +from joblib import dump, load +from joblib.externals import cloudpickle +from numpy.typing import NDArray +from pandas import DataFrame + +from freqtrade.configuration import TimeRange +from freqtrade.data.history import load_pair_history +from freqtrade.exceptions import OperationalException +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.strategy.interface import IStrategy + + +logger = logging.getLogger(__name__) + + +class pair_info(TypedDict): + model_filename: str + first: bool + trained_timestamp: int + priority: int + data_path: str + extras: dict + + +class FreqaiDataDrawer: + """ + Class aimed at holding all pair models/info in memory for better inferencing/retrainig/saving + /loading to/from disk. + This object remains persistent throughout live/dry. + + Record of contribution: + FreqAI was developed by a group of individuals who all contributed specific skillsets to the + project. + + Conception and software development: + Robert Caulk @robcaulk + + Theoretical brainstorming: + Elin Törnquist @th0rntwig + + Code review, software architecture brainstorming: + @xmatthias + + Beta testing and bug reporting: + @bloodhunter4rc, Salah Lamkadem @ikonx, @ken11o2, @longyu, @paranoidandy, @smidelis, @smarm + Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert + """ + + def __init__(self, full_path: Path, config: dict, follow_mode: bool = False): + + self.config = config + self.freqai_info = config.get("freqai", {}) + # dictionary holding all pair metadata necessary to load in from disk + self.pair_dict: Dict[str, pair_info] = {} + # dictionary holding all actively inferenced models in memory given a model filename + self.model_dictionary: Dict[str, Any] = {} + self.model_return_values: Dict[str, DataFrame] = {} + self.historic_data: Dict[str, Dict[str, DataFrame]] = {} + self.historic_predictions: Dict[str, DataFrame] = {} + self.follower_dict: Dict[str, pair_info] = {} + self.full_path = full_path + self.follower_name: str = self.config.get("bot_name", "follower1") + self.follower_dict_path = Path( + self.full_path / f"follower_dictionary-{self.follower_name}.json" + ) + self.historic_predictions_path = Path(self.full_path / "historic_predictions.pkl") + self.pair_dictionary_path = Path(self.full_path / "pair_dictionary.json") + self.follow_mode = follow_mode + if follow_mode: + self.create_follower_dict() + self.load_drawer_from_disk() + self.load_historic_predictions_from_disk() + self.training_queue: Dict[str, int] = {} + self.history_lock = threading.Lock() + self.save_lock = threading.Lock() + self.pair_dict_lock = threading.Lock() + self.old_DBSCAN_eps: Dict[str, float] = {} + self.empty_pair_dict: pair_info = { + "model_filename": "", "trained_timestamp": 0, + "priority": 1, "first": True, "data_path": "", "extras": {}} + + def load_drawer_from_disk(self): + """ + Locate and load a previously saved data drawer full of all pair model metadata in + present model folder. + :return: bool - whether or not the drawer was located + """ + exists = self.pair_dictionary_path.is_file() + if exists: + with open(self.pair_dictionary_path, "r") as fp: + self.pair_dict = json.load(fp) + elif not self.follow_mode: + logger.info("Could not find existing datadrawer, starting from scratch") + else: + logger.warning( + f"Follower could not find pair_dictionary at {self.full_path} " + "sending null values back to strategy" + ) + + return exists + + def load_historic_predictions_from_disk(self): + """ + Locate and load a previously saved historic predictions. + :return: bool - whether or not the drawer was located + """ + exists = self.historic_predictions_path.is_file() + if exists: + with open(self.historic_predictions_path, "rb") as fp: + self.historic_predictions = cloudpickle.load(fp) + logger.info( + f"Found existing historic predictions at {self.full_path}, but beware " + "that statistics may be inaccurate if the bot has been offline for " + "an extended period of time." + ) + elif not self.follow_mode: + logger.info("Could not find existing historic_predictions, starting from scratch") + else: + logger.warning( + f"Follower could not find historic predictions at {self.full_path} " + "sending null values back to strategy" + ) + + return exists + + def save_historic_predictions_to_disk(self): + """ + Save data drawer full of all pair model metadata in present model folder. + """ + with open(self.historic_predictions_path, "wb") as fp: + cloudpickle.dump(self.historic_predictions, fp, protocol=cloudpickle.DEFAULT_PROTOCOL) + + def save_drawer_to_disk(self): + """ + Save data drawer full of all pair model metadata in present model folder. + """ + with self.save_lock: + with open(self.pair_dictionary_path, 'w') as fp: + rapidjson.dump(self.pair_dict, fp, default=self.np_encoder, + number_mode=rapidjson.NM_NATIVE) + + def save_follower_dict_to_disk(self): + """ + Save follower dictionary to disk (used by strategy for persistent prediction targets) + """ + with open(self.follower_dict_path, "w") as fp: + rapidjson.dump(self.follower_dict, fp, default=self.np_encoder, + number_mode=rapidjson.NM_NATIVE) + + def create_follower_dict(self): + """ + Create or dictionary for each follower to maintain unique persistent prediction targets + """ + + whitelist_pairs = self.config.get("exchange", {}).get("pair_whitelist") + + exists = self.follower_dict_path.is_file() + + if exists: + logger.info("Found an existing follower dictionary") + + for pair in whitelist_pairs: + self.follower_dict[pair] = {} + + self.save_follower_dict_to_disk() + + def np_encoder(self, object): + if isinstance(object, np.generic): + return object.item() + + def get_pair_dict_info(self, pair: str) -> Tuple[str, int, bool]: + """ + Locate and load existing model metadata from persistent storage. If not located, + create a new one and append the current pair to it and prepare it for its first + training + :param pair: str: pair to lookup + :return: + model_filename: str = unique filename used for loading persistent objects from disk + trained_timestamp: int = the last time the coin was trained + return_null_array: bool = Follower could not find pair metadata + """ + + pair_dict = self.pair_dict.get(pair) + data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "") + return_null_array = False + + if pair_dict: + model_filename = pair_dict["model_filename"] + trained_timestamp = pair_dict["trained_timestamp"] + elif not self.follow_mode: + self.pair_dict[pair] = self.empty_pair_dict.copy() + model_filename = "" + trained_timestamp = 0 + self.pair_dict[pair]["priority"] = len(self.pair_dict) + + if not data_path_set and self.follow_mode: + logger.warning( + f"Follower could not find current pair {pair} in " + f"pair_dictionary at path {self.full_path}, sending null values " + "back to strategy." + ) + trained_timestamp = 0 + model_filename = '' + return_null_array = True + + return model_filename, trained_timestamp, return_null_array + + def set_pair_dict_info(self, metadata: dict) -> None: + pair_in_dict = self.pair_dict.get(metadata["pair"]) + if pair_in_dict: + return + else: + self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy() + self.pair_dict[metadata["pair"]]["priority"] = len(self.pair_dict) + + return + + def pair_to_end_of_training_queue(self, pair: str) -> None: + # march all pairs up in the queue + with self.pair_dict_lock: + for p in self.pair_dict: + self.pair_dict[p]["priority"] -= 1 + # send pair to end of queue + self.pair_dict[pair]["priority"] = len(self.pair_dict) + + def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None: + """ + Set the initial return values to the historical predictions dataframe. This avoids needing + to repredict on historical candles, and also stores historical predictions despite + retrainings (so stored predictions are true predictions, not just inferencing on trained + data) + """ + + hist_df = self.historic_predictions + len_diff = len(hist_df[pair].index) - len(pred_df.index) + if len_diff < 0: + df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]], + ignore_index=True, keys=hist_df[pair].keys()) + else: + df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True) + df_concat = df_concat.fillna(0) + self.model_return_values[pair] = df_concat + + def append_model_predictions(self, pair: str, predictions: DataFrame, + do_preds: NDArray[np.int_], + dk: FreqaiDataKitchen, len_df: int) -> None: + """ + Append model predictions to historic predictions dataframe, then set the + strategy return dataframe to the tail of the historic predictions. The length of + the tail is equivalent to the length of the dataframe that entered FreqAI from + the strategy originally. Doing this allows FreqUI to always display the correct + historic predictions. + """ + + index = self.historic_predictions[pair].index[-1:] + columns = self.historic_predictions[pair].columns + + nan_df = pd.DataFrame(np.nan, index=index, columns=columns) + self.historic_predictions[pair] = pd.concat( + [self.historic_predictions[pair], nan_df], ignore_index=True, axis=0) + df = self.historic_predictions[pair] + + # model outputs and associated statistics + for label in predictions.columns: + df[label].iloc[-1] = predictions[label].iloc[-1] + if df[label].dtype == object: + continue + df[f"{label}_mean"].iloc[-1] = dk.data["labels_mean"][label] + df[f"{label}_std"].iloc[-1] = dk.data["labels_std"][label] + + # outlier indicators + df["do_predict"].iloc[-1] = do_preds[-1] + if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0: + df["DI_values"].iloc[-1] = dk.DI_values[-1] + + # extra values the user added within custom prediction model + if dk.data['extra_returns_per_train']: + rets = dk.data['extra_returns_per_train'] + for return_str in rets: + df[return_str].iloc[-1] = rets[return_str] + + self.model_return_values[pair] = df.tail(len_df).reset_index(drop=True) + + def attach_return_values_to_return_dataframe( + self, pair: str, dataframe: DataFrame) -> DataFrame: + """ + Attach the return values to the strat dataframe + :param dataframe: DataFrame = strategy dataframe + :return: DataFrame = strat dataframe with return values attached + """ + df = self.model_return_values[pair] + to_keep = [col for col in dataframe.columns if not col.startswith("&")] + dataframe = pd.concat([dataframe[to_keep], df], axis=1) + return dataframe + + def return_null_values_to_strategy(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> None: + """ + Build 0 filled dataframe to return to strategy + """ + + dk.find_features(dataframe) + + full_labels = dk.label_list + dk.unique_class_list + + for label in full_labels: + dataframe[label] = 0 + dataframe[f"{label}_mean"] = 0 + dataframe[f"{label}_std"] = 0 + + dataframe["do_predict"] = 0 + + if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0: + dataframe["DI_values"] = 0 + + if dk.data['extra_returns_per_train']: + rets = dk.data['extra_returns_per_train'] + for return_str in rets: + dataframe[return_str] = 0 + + dk.return_dataframe = dataframe + + def purge_old_models(self) -> None: + + model_folders = [x for x in self.full_path.iterdir() if x.is_dir()] + + pattern = re.compile(r"sub-train-(\w+)_(\d{10})") + + delete_dict: Dict[str, Any] = {} + + for dir in model_folders: + result = pattern.match(str(dir.name)) + if result is None: + break + coin = result.group(1) + timestamp = result.group(2) + + if coin not in delete_dict: + delete_dict[coin] = {} + delete_dict[coin]["num_folders"] = 1 + delete_dict[coin]["timestamps"] = {int(timestamp): dir} + else: + delete_dict[coin]["num_folders"] += 1 + delete_dict[coin]["timestamps"][int(timestamp)] = dir + + for coin in delete_dict: + if delete_dict[coin]["num_folders"] > 2: + sorted_dict = collections.OrderedDict( + sorted(delete_dict[coin]["timestamps"].items()) + ) + num_delete = len(sorted_dict) - 2 + deleted = 0 + for k, v in sorted_dict.items(): + if deleted >= num_delete: + break + logger.info(f"Freqai purging old model file {v}") + shutil.rmtree(v) + deleted += 1 + + def update_follower_metadata(self): + # follower needs to load from disk to get any changes made by leader to pair_dict + self.load_drawer_from_disk() + if self.config.get("freqai", {}).get("purge_old_models", False): + self.purge_old_models() + + # Functions pulled back from FreqaiDataKitchen because they relied on DataDrawer + + def save_data(self, model: Any, coin: str, dk: FreqaiDataKitchen) -> None: + """ + Saves all data associated with a model for a single sub-train time range + :params: + :model: User trained model which can be reused for inferencing to generate + predictions + """ + + if not dk.data_path.is_dir(): + dk.data_path.mkdir(parents=True, exist_ok=True) + + save_path = Path(dk.data_path) + + # Save the trained model + if not dk.keras: + dump(model, save_path / f"{dk.model_filename}_model.joblib") + else: + model.save(save_path / f"{dk.model_filename}_model.h5") + + if dk.svm_model is not None: + dump(dk.svm_model, save_path / f"{dk.model_filename}_svm_model.joblib") + + dk.data["data_path"] = str(dk.data_path) + dk.data["model_filename"] = str(dk.model_filename) + dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns) + dk.data["label_list"] = dk.label_list + # store the metadata + with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp: + rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE) + + # save the train data to file so we can check preds for area of applicability later + dk.data_dictionary["train_features"].to_pickle( + save_path / f"{dk.model_filename}_trained_df.pkl" + ) + + dk.data_dictionary["train_dates"].to_pickle( + save_path / f"{dk.model_filename}_trained_dates_df.pkl" + ) + + if self.freqai_info["feature_parameters"].get("principal_component_analysis"): + cloudpickle.dump( + dk.pca, open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "wb") + ) + + # if self.live: + self.model_dictionary[coin] = model + self.pair_dict[coin]["model_filename"] = dk.model_filename + self.pair_dict[coin]["data_path"] = str(dk.data_path) + self.save_drawer_to_disk() + + return + + def load_data(self, coin: str, dk: FreqaiDataKitchen) -> Any: + """ + loads all data required to make a prediction on a sub-train time range + :returns: + :model: User trained model which can be inferenced for new predictions + """ + + if not self.pair_dict[coin]["model_filename"]: + return None + + if dk.live: + dk.model_filename = self.pair_dict[coin]["model_filename"] + dk.data_path = Path(self.pair_dict[coin]["data_path"]) + if self.freqai_info.get("follow_mode", False): + # follower can be on a different system which is rsynced from the leader: + dk.data_path = Path( + self.config["user_data_dir"] + / "models" + / dk.data_path.parts[-2] + / dk.data_path.parts[-1] + ) + + with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp: + dk.data = json.load(fp) + dk.training_features_list = dk.data["training_features_list"] + dk.label_list = dk.data["label_list"] + + dk.data_dictionary["train_features"] = pd.read_pickle( + dk.data_path / f"{dk.model_filename}_trained_df.pkl" + ) + + # try to access model in memory instead of loading object from disk to save time + if dk.live and coin in self.model_dictionary: + model = self.model_dictionary[coin] + elif not dk.keras: + model = load(dk.data_path / f"{dk.model_filename}_model.joblib") + else: + from tensorflow import keras + + model = keras.models.load_model(dk.data_path / f"{dk.model_filename}_model.h5") + + if Path(dk.data_path / f"{dk.model_filename}_svm_model.joblib").is_file(): + dk.svm_model = load(dk.data_path / f"{dk.model_filename}_svm_model.joblib") + + if not model: + raise OperationalException( + f"Unable to load model, ensure model exists at " f"{dk.data_path} " + ) + + if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]: + dk.pca = cloudpickle.load( + open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb") + ) + + return model + + def update_historic_data(self, strategy: IStrategy, dk: FreqaiDataKitchen) -> None: + """ + Append new candles to our stores historic data (in memory) so that + we do not need to load candle history from disk and we dont need to + pinging exchange multiple times for the same candle. + :params: + dataframe: DataFrame = strategy provided dataframe + """ + feat_params = self.freqai_info["feature_parameters"] + with self.history_lock: + history_data = self.historic_data + + for pair in dk.all_pairs: + for tf in feat_params.get("include_timeframes"): + + # check if newest candle is already appended + df_dp = strategy.dp.get_pair_dataframe(pair, tf) + if len(df_dp.index) == 0: + continue + if str(history_data[pair][tf].iloc[-1]["date"]) == str( + df_dp.iloc[-1:]["date"].iloc[-1] + ): + continue + + try: + index = ( + df_dp.loc[ + df_dp["date"] == history_data[pair][tf].iloc[-1]["date"] + ].index[0] + + 1 + ) + except IndexError: + logger.warning( + f"Unable to update pair history for {pair}. " + "If this does not resolve itself after 1 additional candle, " + "please report the error to #freqai discord channel" + ) + return + + history_data[pair][tf] = pd.concat( + [ + history_data[pair][tf], + df_dp.iloc[index:], + ], + ignore_index=True, + axis=0, + ) + + def load_all_pair_histories(self, timerange: TimeRange, dk: FreqaiDataKitchen) -> None: + """ + Load pair histories for all whitelist and corr_pairlist pairs. + Only called once upon startup of bot. + :params: + timerange: TimeRange = full timerange required to populate all indicators + for training according to user defined train_period_days + """ + history_data = self.historic_data + + for pair in dk.all_pairs: + if pair not in history_data: + history_data[pair] = {} + for tf in self.freqai_info["feature_parameters"].get("include_timeframes"): + history_data[pair][tf] = load_pair_history( + datadir=self.config["datadir"], + timeframe=tf, + pair=pair, + timerange=timerange, + data_format=self.config.get("dataformat_ohlcv", "json"), + candle_type=self.config.get("trading_mode", "spot"), + ) + + def get_base_and_corr_dataframes( + self, timerange: TimeRange, pair: str, dk: FreqaiDataKitchen + ) -> Tuple[Dict[Any, Any], Dict[Any, Any]]: + """ + Searches through our historic_data in memory and returns the dataframes relevant + to the present pair. + :params: + timerange: TimeRange = full timerange required to populate all indicators + for training according to user defined train_period_days + metadata: dict = strategy furnished pair metadata + """ + with self.history_lock: + corr_dataframes: Dict[Any, Any] = {} + base_dataframes: Dict[Any, Any] = {} + historic_data = self.historic_data + pairs = self.freqai_info["feature_parameters"].get( + "include_corr_pairlist", [] + ) + + for tf in self.freqai_info["feature_parameters"].get("include_timeframes"): + base_dataframes[tf] = dk.slice_dataframe(timerange, historic_data[pair][tf]) + if pairs: + for p in pairs: + if pair in p: + continue # dont repeat anything from whitelist + if p not in corr_dataframes: + corr_dataframes[p] = {} + corr_dataframes[p][tf] = dk.slice_dataframe( + timerange, historic_data[p][tf] + ) + + return corr_dataframes, base_dataframes + + # to be used if we want to send predictions directly to the follower instead of forcing + # follower to load models and inference + # def save_model_return_values_to_disk(self) -> None: + # with open(self.full_path / str('model_return_values.json'), "w") as fp: + # json.dump(self.model_return_values, fp, default=self.np_encoder) + + # def load_model_return_values_from_disk(self, dk: FreqaiDataKitchen) -> FreqaiDataKitchen: + # exists = Path(self.full_path / str('model_return_values.json')).resolve().exists() + # if exists: + # with open(self.full_path / str('model_return_values.json'), "r") as fp: + # self.model_return_values = json.load(fp) + # elif not self.follow_mode: + # logger.info("Could not find existing datadrawer, starting from scratch") + # else: + # logger.warning(f'Follower could not find pair_dictionary at {self.full_path} ' + # 'sending null values back to strategy') + + # return exists, dk diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py new file mode 100644 index 000000000..8e68c9a38 --- /dev/null +++ b/freqtrade/freqai/data_kitchen.py @@ -0,0 +1,1088 @@ +import copy +import datetime +import logging +import shutil +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import numpy as np +import numpy.typing as npt +import pandas as pd +from pandas import DataFrame +from sklearn import linear_model +from sklearn.cluster import DBSCAN +from sklearn.metrics.pairwise import pairwise_distances +from sklearn.model_selection import train_test_split +from sklearn.neighbors import NearestNeighbors + +from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider +from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_seconds +from freqtrade.strategy.interface import IStrategy + + +SECONDS_IN_DAY = 86400 +SECONDS_IN_HOUR = 3600 + +logger = logging.getLogger(__name__) + + +class FreqaiDataKitchen: + """ + Class designed to analyze data for a single pair. Employed by the IFreqaiModel class. + Functionalities include holding, saving, loading, and analyzing the data. + + This object is not persistent, it is reinstantiated for each coin, each time the coin + model needs to be inferenced or trained. + + Record of contribution: + FreqAI was developed by a group of individuals who all contributed specific skillsets to the + project. + + Conception and software development: + Robert Caulk @robcaulk + + Theoretical brainstorming: + Elin Törnquist @th0rntwig + + Code review, software architecture brainstorming: + @xmatthias + + Beta testing and bug reporting: + @bloodhunter4rc, Salah Lamkadem @ikonx, @ken11o2, @longyu, @paranoidandy, @smidelis, @smarm + Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert + """ + + def __init__( + self, + config: Dict[str, Any], + live: bool = False, + pair: str = "", + ): + self.data: Dict[str, Any] = {} + self.data_dictionary: Dict[str, DataFrame] = {} + self.config = config + self.freqai_config: Dict[str, Any] = config["freqai"] + self.full_df: DataFrame = DataFrame() + self.append_df: DataFrame = DataFrame() + self.data_path = Path() + self.label_list: List = [] + self.training_features_list: List = [] + self.model_filename: str = "" + self.live = live + self.pair = pair + + self.svm_model: linear_model.SGDOneClassSVM = None + self.keras: bool = self.freqai_config.get("keras", False) + self.set_all_pairs() + if not self.live: + if not self.config["timerange"]: + raise OperationalException( + 'Please pass --timerange if you intend to use FreqAI for backtesting.') + self.full_timerange = self.create_fulltimerange( + self.config["timerange"], self.freqai_config.get("train_period_days", 0) + ) + + (self.training_timeranges, self.backtesting_timeranges) = self.split_timerange( + self.full_timerange, + config["freqai"]["train_period_days"], + config["freqai"]["backtest_period_days"], + ) + + self.data['extra_returns_per_train'] = self.freqai_config.get('extra_returns_per_train', {}) + self.thread_count = self.freqai_config.get("data_kitchen_thread_count", -1) + self.train_dates: DataFrame = pd.DataFrame() + self.unique_classes: Dict[str, list] = {} + self.unique_class_list: list = [] + + def set_paths( + self, + pair: str, + trained_timestamp: int = None, + ) -> None: + """ + Set the paths to the data for the present coin/botloop + :params: + metadata: dict = strategy furnished pair metadata + trained_timestamp: int = timestamp of most recent training + """ + self.full_path = Path( + self.config["user_data_dir"] / "models" / str(self.freqai_config.get("identifier")) + ) + + self.data_path = Path( + self.full_path + / f"sub-train-{pair.split('/')[0]}_{trained_timestamp}" + ) + + return + + def make_train_test_datasets( + self, filtered_dataframe: DataFrame, labels: DataFrame + ) -> Dict[Any, Any]: + """ + Given the dataframe for the full history for training, split the data into + training and test data according to user specified parameters in configuration + file. + :filtered_dataframe: cleaned dataframe ready to be split. + :labels: cleaned labels ready to be split. + """ + feat_dict = self.freqai_config["feature_parameters"] + + weights: npt.ArrayLike + if feat_dict.get("weight_factor", 0) > 0: + weights = self.set_weights_higher_recent(len(filtered_dataframe)) + else: + weights = np.ones(len(filtered_dataframe)) + + if feat_dict.get("stratify_training_data", 0) > 0: + stratification = np.zeros(len(filtered_dataframe)) + for i in range(1, len(stratification)): + if i % feat_dict.get("stratify_training_data", 0) == 0: + stratification[i] = 1 + else: + stratification = None + + if self.freqai_config.get('data_split_parameters', {}).get('test_size', 0.1) != 0: + ( + train_features, + test_features, + train_labels, + test_labels, + train_weights, + test_weights, + ) = train_test_split( + filtered_dataframe[: filtered_dataframe.shape[0]], + labels, + weights, + stratify=stratification, + **self.config["freqai"]["data_split_parameters"], + ) + else: + test_labels = np.zeros(2) + test_features = pd.DataFrame() + test_weights = np.zeros(2) + train_features = filtered_dataframe + train_labels = labels + train_weights = weights + + return self.build_data_dictionary( + train_features, test_features, train_labels, test_labels, train_weights, test_weights + ) + + def filter_features( + self, + unfiltered_dataframe: DataFrame, + training_feature_list: List, + label_list: List = list(), + training_filter: bool = True, + ) -> Tuple[DataFrame, DataFrame]: + """ + Filter the unfiltered dataframe to extract the user requested features/labels and properly + remove all NaNs. Any row with a NaN is removed from training dataset or replaced with + 0s in the prediction dataset. However, prediction dataset do_predict will reflect any + row that had a NaN and will shield user from that prediction. + :params: + :unfiltered_dataframe: the full dataframe for the present training period + :training_feature_list: list, the training feature list constructed by + self.build_feature_list() according to user specified parameters in the configuration file. + :labels: the labels for the dataset + :training_filter: boolean which lets the function know if it is training data or + prediction data to be filtered. + :returns: + :filtered_dataframe: dataframe cleaned of NaNs and only containing the user + requested feature set. + :labels: labels cleaned of NaNs. + """ + filtered_dataframe = unfiltered_dataframe.filter(training_feature_list, axis=1) + filtered_dataframe = filtered_dataframe.replace([np.inf, -np.inf], np.nan) + + drop_index = pd.isnull(filtered_dataframe).any(1) # get the rows that have NaNs, + drop_index = drop_index.replace(True, 1).replace(False, 0) # pep8 requirement. + if (training_filter): + # we don't care about total row number (total no. datapoints) in training, we only care + # about removing any row with NaNs + # if labels has multiple columns (user wants to train multiple modelEs), we detect here + labels = unfiltered_dataframe.filter(label_list, axis=1) + drop_index_labels = pd.isnull(labels).any(1) + drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0) + dates = unfiltered_dataframe['date'] + filtered_dataframe = filtered_dataframe[ + (drop_index == 0) & (drop_index_labels == 0) + ] # dropping values + labels = labels[ + (drop_index == 0) & (drop_index_labels == 0) + ] # assuming the labels depend entirely on the dataframe here. + self.train_dates = dates[ + (drop_index == 0) & (drop_index_labels == 0) + ] + logger.info( + f"dropped {len(unfiltered_dataframe) - len(filtered_dataframe)} training points" + f" due to NaNs in populated dataset {len(unfiltered_dataframe)}." + ) + if (1 - len(filtered_dataframe) / len(unfiltered_dataframe)) > 0.1 and self.live: + worst_indicator = str(unfiltered_dataframe.count().idxmin()) + logger.warning( + f" {(1 - len(filtered_dataframe)/len(unfiltered_dataframe)) * 100:.0f} percent " + " of training data dropped due to NaNs, model may perform inconsistent " + f"with expectations. Verify {worst_indicator}" + ) + self.data["filter_drop_index_training"] = drop_index + + else: + # we are backtesting so we need to preserve row number to send back to strategy, + # so now we use do_predict to avoid any prediction based on a NaN + drop_index = pd.isnull(filtered_dataframe).any(1) + self.data["filter_drop_index_prediction"] = drop_index + filtered_dataframe.fillna(0, inplace=True) + # replacing all NaNs with zeros to avoid issues in 'prediction', but any prediction + # that was based on a single NaN is ultimately protected from buys with do_predict + drop_index = ~drop_index + self.do_predict = np.array(drop_index.replace(True, 1).replace(False, 0)) + if (len(self.do_predict) - self.do_predict.sum()) > 0: + logger.info( + "dropped %s of %s prediction data points due to NaNs.", + len(self.do_predict) - self.do_predict.sum(), + len(filtered_dataframe), + ) + labels = [] + + return filtered_dataframe, labels + + def build_data_dictionary( + self, + train_df: DataFrame, + test_df: DataFrame, + train_labels: DataFrame, + test_labels: DataFrame, + train_weights: Any, + test_weights: Any, + ) -> Dict: + + self.data_dictionary = { + "train_features": train_df, + "test_features": test_df, + "train_labels": train_labels, + "test_labels": test_labels, + "train_weights": train_weights, + "test_weights": test_weights, + "train_dates": self.train_dates + } + + return self.data_dictionary + + def normalize_data(self, data_dictionary: Dict) -> Dict[Any, Any]: + """ + Normalize all data in the data_dictionary according to the training dataset + :params: + :data_dictionary: dictionary containing the cleaned and split training/test data/labels + :returns: + :data_dictionary: updated dictionary with standardized values. + """ + # standardize the data by training stats + train_max = data_dictionary["train_features"].max() + train_min = data_dictionary["train_features"].min() + data_dictionary["train_features"] = ( + 2 * (data_dictionary["train_features"] - train_min) / (train_max - train_min) - 1 + ) + data_dictionary["test_features"] = ( + 2 * (data_dictionary["test_features"] - train_min) / (train_max - train_min) - 1 + ) + + for item in train_max.keys(): + self.data[item + "_max"] = train_max[item] + self.data[item + "_min"] = train_min[item] + + for item in data_dictionary["train_labels"].keys(): + if data_dictionary["train_labels"][item].dtype == object: + continue + train_labels_max = data_dictionary["train_labels"][item].max() + train_labels_min = data_dictionary["train_labels"][item].min() + data_dictionary["train_labels"][item] = ( + 2 + * (data_dictionary["train_labels"][item] - train_labels_min) + / (train_labels_max - train_labels_min) + - 1 + ) + if self.freqai_config.get('data_split_parameters', {}).get('test_size', 0.1) != 0: + data_dictionary["test_labels"][item] = ( + 2 + * (data_dictionary["test_labels"][item] - train_labels_min) + / (train_labels_max - train_labels_min) + - 1 + ) + + self.data[f"{item}_max"] = train_labels_max # .to_dict() + self.data[f"{item}_min"] = train_labels_min # .to_dict() + return data_dictionary + + def normalize_data_from_metadata(self, df: DataFrame) -> DataFrame: + """ + Normalize a set of data using the mean and standard deviation from + the associated training data. + :param df: Dataframe to be standardized + """ + + for item in df.keys(): + df[item] = ( + 2 + * (df[item] - self.data[f"{item}_min"]) + / (self.data[f"{item}_max"] - self.data[f"{item}_min"]) + - 1 + ) + + return df + + def denormalize_labels_from_metadata(self, df: DataFrame) -> DataFrame: + """ + Normalize a set of data using the mean and standard deviation from + the associated training data. + :param df: Dataframe of predictions to be denormalized + """ + + for label in df.columns: + if df[label].dtype == object or label in self.unique_class_list: + continue + df[label] = ( + (df[label] + 1) + * (self.data[f"{label}_max"] - self.data[f"{label}_min"]) + / 2 + ) + self.data[f"{label}_min"] + + return df + + def split_timerange( + self, tr: str, train_split: int = 28, bt_split: float = 7 + ) -> Tuple[list, list]: + """ + Function which takes a single time range (tr) and splits it + into sub timeranges to train and backtest on based on user input + tr: str, full timerange to train on + train_split: the period length for the each training (days). Specified in user + configuration file + bt_split: the backtesting length (days). Specified in user configuration file + """ + + if not isinstance(train_split, int) or train_split < 1: + raise OperationalException( + f"train_period_days must be an integer greater than 0. Got {train_split}." + ) + train_period_days = train_split * SECONDS_IN_DAY + bt_period = bt_split * SECONDS_IN_DAY + + full_timerange = TimeRange.parse_timerange(tr) + config_timerange = TimeRange.parse_timerange(self.config["timerange"]) + if config_timerange.stopts == 0: + config_timerange.stopts = int( + datetime.datetime.now(tz=datetime.timezone.utc).timestamp() + ) + timerange_train = copy.deepcopy(full_timerange) + timerange_backtest = copy.deepcopy(full_timerange) + + tr_training_list = [] + tr_backtesting_list = [] + tr_training_list_timerange = [] + tr_backtesting_list_timerange = [] + first = True + + while True: + if not first: + timerange_train.startts = timerange_train.startts + int(bt_period) + timerange_train.stopts = timerange_train.startts + train_period_days + + first = False + start = datetime.datetime.utcfromtimestamp(timerange_train.startts) + stop = datetime.datetime.utcfromtimestamp(timerange_train.stopts) + tr_training_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")) + tr_training_list_timerange.append(copy.deepcopy(timerange_train)) + + # associated backtest period + + timerange_backtest.startts = timerange_train.stopts + + timerange_backtest.stopts = timerange_backtest.startts + int(bt_period) + + if timerange_backtest.stopts > config_timerange.stopts: + timerange_backtest.stopts = config_timerange.stopts + + start = datetime.datetime.utcfromtimestamp(timerange_backtest.startts) + stop = datetime.datetime.utcfromtimestamp(timerange_backtest.stopts) + tr_backtesting_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")) + tr_backtesting_list_timerange.append(copy.deepcopy(timerange_backtest)) + + # ensure we are predicting on exactly same amount of data as requested by user defined + # --timerange + if timerange_backtest.stopts == config_timerange.stopts: + break + + # print(tr_training_list, tr_backtesting_list) + return tr_training_list_timerange, tr_backtesting_list_timerange + + def slice_dataframe(self, timerange: TimeRange, df: DataFrame) -> DataFrame: + """ + Given a full dataframe, extract the user desired window + :param tr: timerange string that we wish to extract from df + :param df: Dataframe containing all candles to run the entire backtest. Here + it is sliced down to just the present training period. + """ + + start = datetime.datetime.fromtimestamp(timerange.startts, tz=datetime.timezone.utc) + stop = datetime.datetime.fromtimestamp(timerange.stopts, tz=datetime.timezone.utc) + df = df.loc[df["date"] >= start, :] + df = df.loc[df["date"] <= stop, :] + + return df + + def principal_component_analysis(self) -> None: + """ + Performs Principal Component Analysis on the data for dimensionality reduction + and outlier detection (see self.remove_outliers()) + No parameters or returns, it acts on the data_dictionary held by the DataHandler. + """ + + from sklearn.decomposition import PCA # avoid importing if we dont need it + + n_components = self.data_dictionary["train_features"].shape[1] + pca = PCA(n_components=n_components) + pca = pca.fit(self.data_dictionary["train_features"]) + n_keep_components = np.argmin(pca.explained_variance_ratio_.cumsum() < 0.999) + pca2 = PCA(n_components=n_keep_components) + self.data["n_kept_components"] = n_keep_components + pca2 = pca2.fit(self.data_dictionary["train_features"]) + logger.info("reduced feature dimension by %s", n_components - n_keep_components) + logger.info("explained variance %f", np.sum(pca2.explained_variance_ratio_)) + train_components = pca2.transform(self.data_dictionary["train_features"]) + + self.data_dictionary["train_features"] = pd.DataFrame( + data=train_components, + columns=["PC" + str(i) for i in range(0, n_keep_components)], + index=self.data_dictionary["train_features"].index, + ) + + # keeping a copy of the non-transformed features so we can check for errors during + # model load from disk + self.data["training_features_list_raw"] = copy.deepcopy(self.training_features_list) + self.training_features_list = self.data_dictionary["train_features"].columns + + if self.freqai_config.get('data_split_parameters', {}).get('test_size', 0.1) != 0: + test_components = pca2.transform(self.data_dictionary["test_features"]) + self.data_dictionary["test_features"] = pd.DataFrame( + data=test_components, + columns=["PC" + str(i) for i in range(0, n_keep_components)], + index=self.data_dictionary["test_features"].index, + ) + + self.data["n_kept_components"] = n_keep_components + self.pca = pca2 + + logger.info(f"PCA reduced total features from {n_components} to {n_keep_components}") + + if not self.data_path.is_dir(): + self.data_path.mkdir(parents=True, exist_ok=True) + + return None + + def pca_transform(self, filtered_dataframe: DataFrame) -> None: + """ + Use an existing pca transform to transform data into components + :params: + filtered_dataframe: DataFrame = the cleaned dataframe + """ + pca_components = self.pca.transform(filtered_dataframe) + self.data_dictionary["prediction_features"] = pd.DataFrame( + data=pca_components, + columns=["PC" + str(i) for i in range(0, self.data["n_kept_components"])], + index=filtered_dataframe.index, + ) + + def compute_distances(self) -> float: + """ + Compute distances between each training point and every other training + point. This metric defines the neighborhood of trained data and is used + for prediction confidence in the Dissimilarity Index + """ + # logger.info("computing average mean distance for all training points") + pairwise = pairwise_distances( + self.data_dictionary["train_features"], n_jobs=self.thread_count) + # remove the diagonal distances which are itself distances ~0 + np.fill_diagonal(pairwise, np.NaN) + pairwise = pairwise.reshape(-1, 1) + avg_mean_dist = pairwise[~np.isnan(pairwise)].mean() + + return avg_mean_dist + + def get_outlier_percentage(self, dropped_pts: npt.NDArray) -> float: + """ + Check if more than X% of points werer dropped during outlier detection. + """ + outlier_protection_pct = self.freqai_config["feature_parameters"].get( + "outlier_protection_percentage", 30) + outlier_pct = (dropped_pts.sum() / len(dropped_pts)) * 100 + if outlier_pct >= outlier_protection_pct: + self.svm_model = None + return outlier_pct + else: + return 0.0 + + def use_SVM_to_remove_outliers(self, predict: bool) -> None: + """ + Build/inference a Support Vector Machine to detect outliers + in training data and prediction + :params: + predict: bool = If true, inference an existing SVM model, else construct one + """ + + if self.keras: + logger.warning( + "SVM outlier removal not currently supported for Keras based models. " + "Skipping user requested function." + ) + if predict: + self.do_predict = np.ones(len(self.data_dictionary["prediction_features"])) + return + + if predict: + if not self.svm_model: + logger.warning("No svm model available for outlier removal") + return + y_pred = self.svm_model.predict(self.data_dictionary["prediction_features"]) + do_predict = np.where(y_pred == -1, 0, y_pred) + + if (len(do_predict) - do_predict.sum()) > 0: + logger.info(f"SVM tossed {len(do_predict) - do_predict.sum()} predictions.") + self.do_predict += do_predict + self.do_predict -= 1 + + else: + # use SGDOneClassSVM to increase speed? + svm_params = self.freqai_config["feature_parameters"].get( + "svm_params", {"shuffle": False, "nu": 0.1}) + self.svm_model = linear_model.SGDOneClassSVM(**svm_params).fit( + self.data_dictionary["train_features"] + ) + y_pred = self.svm_model.predict(self.data_dictionary["train_features"]) + kept_points = np.where(y_pred == -1, 0, y_pred) + # keep_index = np.where(y_pred == 1) + outlier_pct = self.get_outlier_percentage(1 - kept_points) + if outlier_pct: + logger.warning( + f"SVM detected {outlier_pct:.2f}% of the points as outliers. " + f"Keeping original dataset." + ) + return + + self.data_dictionary["train_features"] = self.data_dictionary["train_features"][ + (y_pred == 1) + ] + self.data_dictionary["train_labels"] = self.data_dictionary["train_labels"][ + (y_pred == 1) + ] + self.data_dictionary["train_weights"] = self.data_dictionary["train_weights"][ + (y_pred == 1) + ] + + logger.info( + f"SVM tossed {len(y_pred) - kept_points.sum()}" + f" train points from {len(y_pred)} total points." + ) + + # same for test data + # TODO: This (and the part above) could be refactored into a separate function + # to reduce code duplication + if self.freqai_config['data_split_parameters'].get('test_size', 0.1) != 0: + y_pred = self.svm_model.predict(self.data_dictionary["test_features"]) + kept_points = np.where(y_pred == -1, 0, y_pred) + self.data_dictionary["test_features"] = self.data_dictionary["test_features"][ + (y_pred == 1) + ] + self.data_dictionary["test_labels"] = self.data_dictionary["test_labels"][( + y_pred == 1)] + self.data_dictionary["test_weights"] = self.data_dictionary["test_weights"][ + (y_pred == 1) + ] + + logger.info( + f"SVM tossed {len(y_pred) - kept_points.sum()}" + f" test points from {len(y_pred)} total points." + ) + + return + + def use_DBSCAN_to_remove_outliers(self, predict: bool, eps=None) -> None: + """ + Use DBSCAN to cluster training data and remove "noisy" data (read outliers). + User controls this via the config param `DBSCAN_outlier_pct` which indicates the + pct of training data that they want to be considered outliers. + :params: + predict: bool = If False (training), iterate to find the best hyper parameters to match + user requested outlier percent target. If True (prediction), use the parameters + determined from the previous training to estimate if the current prediction point + is an outlier. + """ + + from math import cos, sin + + if predict: + train_ft_df = self.data_dictionary['train_features'] + pred_ft_df = self.data_dictionary['prediction_features'] + num_preds = len(pred_ft_df) + df = pd.concat([train_ft_df, pred_ft_df], axis=0, ignore_index=True) + clustering = DBSCAN(eps=self.data['DBSCAN_eps'], + min_samples=self.data['DBSCAN_min_samples'], + n_jobs=self.thread_count + ).fit(df) + do_predict = np.where(clustering.labels_[-num_preds:] == -1, 0, 1) + + if (len(do_predict) - do_predict.sum()) > 0: + logger.info(f"DBSCAN tossed {len(do_predict) - do_predict.sum()} predictions") + self.do_predict += do_predict + self.do_predict -= 1 + + else: + + def normalise_distances(distances): + normalised_distances = (distances - distances.min()) / \ + (distances.max() - distances.min()) + return normalised_distances + + def rotate_point(origin, point, angle): + # rotate a point counterclockwise by a given angle (in radians) + # around a given origin + x = origin[0] + cos(angle) * (point[0] - origin[0]) - \ + sin(angle) * (point[1] - origin[1]) + y = origin[1] + sin(angle) * (point[0] - origin[0]) + \ + cos(angle) * (point[1] - origin[1]) + return (x, y) + + MinPts = int(len(self.data_dictionary['train_features'].index) * 0.25) + # measure pairwise distances to nearest neighbours + neighbors = NearestNeighbors( + n_neighbors=MinPts, n_jobs=self.thread_count) + neighbors_fit = neighbors.fit(self.data_dictionary['train_features']) + distances, _ = neighbors_fit.kneighbors(self.data_dictionary['train_features']) + distances = np.sort(distances, axis=0).mean(axis=1) + + normalised_distances = normalise_distances(distances) + x_range = np.linspace(0, 1, len(distances)) + line = np.linspace(normalised_distances[0], + normalised_distances[-1], len(normalised_distances)) + deflection = np.abs(normalised_distances - line) + max_deflection_loc = np.where(deflection == deflection.max())[0][0] + origin = x_range[max_deflection_loc], line[max_deflection_loc] + point = x_range[max_deflection_loc], normalised_distances[max_deflection_loc] + rot_angle = np.pi / 4 + elbow_loc = rotate_point(origin, point, rot_angle) + + epsilon = elbow_loc[1] * (distances[-1] - distances[0]) + distances[0] + + clustering = DBSCAN(eps=epsilon, min_samples=MinPts, + n_jobs=int(self.thread_count)).fit( + self.data_dictionary['train_features'] + ) + + logger.info(f'DBSCAN found eps of {epsilon:.2f}.') + + self.data['DBSCAN_eps'] = epsilon + self.data['DBSCAN_min_samples'] = MinPts + dropped_points = np.where(clustering.labels_ == -1, 1, 0) + + outlier_pct = self.get_outlier_percentage(dropped_points) + if outlier_pct: + logger.warning( + f"DBSCAN detected {outlier_pct:.2f}% of the points as outliers. " + f"Keeping original dataset." + ) + return + + self.data_dictionary['train_features'] = self.data_dictionary['train_features'][ + (clustering.labels_ != -1) + ] + self.data_dictionary["train_labels"] = self.data_dictionary["train_labels"][ + (clustering.labels_ != -1) + ] + self.data_dictionary["train_weights"] = self.data_dictionary["train_weights"][ + (clustering.labels_ != -1) + ] + + logger.info( + f"DBSCAN tossed {dropped_points.sum()}" + f" train points from {len(clustering.labels_)}" + ) + + return + + def find_features(self, dataframe: DataFrame) -> None: + """ + Find features in the strategy provided dataframe + :param dataframe: DataFrame = strategy provided dataframe + :return: + features: list = the features to be used for training/prediction + """ + column_names = dataframe.columns + features = [c for c in column_names if "%" in c] + labels = [c for c in column_names if "&" in c] + if not features: + raise OperationalException("Could not find any features!") + + self.training_features_list = features + self.label_list = labels + + def check_if_pred_in_training_spaces(self) -> None: + """ + Compares the distance from each prediction point to each training data + point. It uses this information to estimate a Dissimilarity Index (DI) + and avoid making predictions on any points that are too far away + from the training data set. + """ + + distance = pairwise_distances( + self.data_dictionary["train_features"], + self.data_dictionary["prediction_features"], + n_jobs=self.thread_count, + ) + + self.DI_values = distance.min(axis=0) / self.data["avg_mean_dist"] + + do_predict = np.where( + self.DI_values < self.freqai_config["feature_parameters"]["DI_threshold"], + 1, + 0, + ) + + outlier_pct = self.get_outlier_percentage(1 - do_predict) + if outlier_pct: + logger.warning( + f"DI detected {outlier_pct:.2f}% of the points as outliers. " + f"Keeping original dataset." + ) + return + + if (len(do_predict) - do_predict.sum()) > 0: + logger.info( + f"DI tossed {len(do_predict) - do_predict.sum()} predictions for " + "being too far from training data" + ) + + self.do_predict += do_predict + self.do_predict -= 1 + + def set_weights_higher_recent(self, num_weights: int) -> npt.ArrayLike: + """ + Set weights so that recent data is more heavily weighted during + training than older data. + """ + wfactor = self.config["freqai"]["feature_parameters"]["weight_factor"] + weights = np.exp(-np.arange(num_weights) / (wfactor * num_weights))[::-1] + return weights + + def append_predictions(self, predictions: DataFrame, do_predict: npt.ArrayLike) -> None: + """ + Append backtest prediction from current backtest period to all previous periods + """ + + append_df = DataFrame() + for label in predictions.columns: + append_df[label] = predictions[label] + if append_df[label].dtype == object: + continue + append_df[f"{label}_mean"] = self.data["labels_mean"][label] + append_df[f"{label}_std"] = self.data["labels_std"][label] + + append_df["do_predict"] = do_predict + if self.freqai_config["feature_parameters"].get("DI_threshold", 0) > 0: + append_df["DI_values"] = self.DI_values + + if self.full_df.empty: + self.full_df = append_df + else: + self.full_df = pd.concat([self.full_df, append_df], axis=0) + + return + + def fill_predictions(self, dataframe): + """ + Back fill values to before the backtesting range so that the dataframe matches size + when it goes back to the strategy. These rows are not included in the backtest. + """ + + len_filler = len(dataframe) - len(self.full_df.index) # startup_candle_count + filler_df = pd.DataFrame( + np.zeros((len_filler, len(self.full_df.columns))), columns=self.full_df.columns + ) + + self.full_df = pd.concat([filler_df, self.full_df], axis=0, ignore_index=True) + + to_keep = [col for col in dataframe.columns if not col.startswith("&")] + self.return_dataframe = pd.concat([dataframe[to_keep], self.full_df], axis=1) + + self.full_df = DataFrame() + + return + + def create_fulltimerange(self, backtest_tr: str, backtest_period_days: int) -> str: + + if not isinstance(backtest_period_days, int): + raise OperationalException("backtest_period_days must be an integer") + + if backtest_period_days < 0: + raise OperationalException("backtest_period_days must be positive") + + backtest_timerange = TimeRange.parse_timerange(backtest_tr) + + if backtest_timerange.stopts == 0: + # typically open ended time ranges do work, however, there are some edge cases where + # it does not. accommodating these kinds of edge cases just to allow open-ended + # timerange is not high enough priority to warrant the effort. It is safer for now + # to simply ask user to add their end date + raise OperationalException("FreqAI backtesting does not allow open ended timeranges. " + "Please indicate the end date of your desired backtesting. " + "timerange.") + # backtest_timerange.stopts = int( + # datetime.datetime.now(tz=datetime.timezone.utc).timestamp() + # ) + + backtest_timerange.startts = ( + backtest_timerange.startts - backtest_period_days * SECONDS_IN_DAY + ) + start = datetime.datetime.utcfromtimestamp(backtest_timerange.startts) + stop = datetime.datetime.utcfromtimestamp(backtest_timerange.stopts) + full_timerange = start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d") + + self.full_path = Path( + self.config["user_data_dir"] / "models" / f"{self.freqai_config['identifier']}" + ) + + config_path = Path(self.config["config_files"][0]) + + if not self.full_path.is_dir(): + self.full_path.mkdir(parents=True, exist_ok=True) + shutil.copy( + config_path.resolve(), + Path(self.full_path / config_path.parts[-1]), + ) + + return full_timerange + + def check_if_model_expired(self, trained_timestamp: int) -> bool: + """ + A model age checker to determine if the model is trustworthy based on user defined + `expiration_hours` in the configuration file. + :param trained_timestamp: int = The time of training for the most recent model. + :return: + bool = If the model is expired or not. + """ + time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() + elapsed_time = (time - trained_timestamp) / 3600 # hours + max_time = self.freqai_config.get("expiration_hours", 0) + if max_time > 0: + return elapsed_time > max_time + else: + return False + + def check_if_new_training_required( + self, trained_timestamp: int + ) -> Tuple[bool, TimeRange, TimeRange]: + + time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() + trained_timerange = TimeRange() + data_load_timerange = TimeRange() + + timeframes = self.freqai_config["feature_parameters"].get("include_timeframes") + + max_tf_seconds = 0 + for tf in timeframes: + secs = timeframe_to_seconds(tf) + if secs > max_tf_seconds: + max_tf_seconds = secs + + # We notice that users like to use exotic indicators where + # they do not know the required timeperiod. Here we include a factor + # of safety by multiplying the user considered "max" by 2. + max_period = self.freqai_config["feature_parameters"].get( + "indicator_max_period_candles", 20 + ) * 2 + additional_seconds = max_period * max_tf_seconds + + if trained_timestamp != 0: + elapsed_time = (time - trained_timestamp) / SECONDS_IN_HOUR + retrain = elapsed_time > self.freqai_config.get("live_retrain_hours", 0) + if retrain: + trained_timerange.startts = int( + time - self.freqai_config.get("train_period_days", 0) * SECONDS_IN_DAY + ) + trained_timerange.stopts = int(time) + # we want to load/populate indicators on more data than we plan to train on so + # because most of the indicators have a rolling timeperiod, and are thus NaNs + # unless they have data further back in time before the start of the train period + data_load_timerange.startts = int( + time + - self.freqai_config.get("train_period_days", 0) * SECONDS_IN_DAY + - additional_seconds + ) + data_load_timerange.stopts = int(time) + else: # user passed no live_trained_timerange in config + trained_timerange.startts = int( + time - self.freqai_config.get("train_period_days", 0) * SECONDS_IN_DAY + ) + trained_timerange.stopts = int(time) + + data_load_timerange.startts = int( + time + - self.freqai_config.get("train_period_days", 0) * SECONDS_IN_DAY + - additional_seconds + ) + data_load_timerange.stopts = int(time) + retrain = True + + return retrain, trained_timerange, data_load_timerange + + def set_new_model_names(self, pair: str, trained_timerange: TimeRange): + + coin, _ = pair.split("/") + self.data_path = Path( + self.full_path + / f"sub-train-{pair.split('/')[0]}_{int(trained_timerange.stopts)}" + ) + + self.model_filename = f"cb_{coin.lower()}_{int(trained_timerange.stopts)}" + + def download_all_data_for_training(self, timerange: TimeRange, dp: DataProvider) -> None: + """ + Called only once upon start of bot to download the necessary data for + populating indicators and training the model. + :param timerange: TimeRange = The full data timerange for populating the indicators + and training the model. + :param dp: DataProvider instance attached to the strategy + """ + new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) + if not dp._exchange: + # Not realistic - this is only called in live mode. + raise OperationalException("Dataprovider did not have an exchange attached.") + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=self.all_pairs, + timeframes=self.freqai_config["feature_parameters"].get("include_timeframes"), + datadir=self.config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=self.config.get("dataformat_ohlcv", "json"), + trading_mode=self.config.get("trading_mode", "spot"), + prepend=self.config.get("prepend_data", False), + ) + + def set_all_pairs(self) -> None: + + self.all_pairs = copy.deepcopy( + self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) + ) + for pair in self.config.get("exchange", "").get("pair_whitelist"): + if pair not in self.all_pairs: + self.all_pairs.append(pair) + + def use_strategy_to_populate_indicators( + self, + strategy: IStrategy, + corr_dataframes: dict = {}, + base_dataframes: dict = {}, + pair: str = "", + prediction_dataframe: DataFrame = pd.DataFrame(), + ) -> DataFrame: + """ + Use the user defined strategy for populating indicators during + retrain + :params: + strategy: IStrategy = user defined strategy object + corr_dataframes: dict = dict containing the informative pair dataframes + (for user defined timeframes) + base_dataframes: dict = dict containing the current pair dataframes + (for user defined timeframes) + metadata: dict = strategy furnished pair metadata + :returns: + dataframe: DataFrame = dataframe containing populated indicators + """ + + # for prediction dataframe creation, we let dataprovider handle everything in the strategy + # so we create empty dictionaries, which allows us to pass None to + # `populate_any_indicators()`. Signaling we want the dp to give us the live dataframe. + tfs = self.freqai_config["feature_parameters"].get("include_timeframes") + pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) + if not prediction_dataframe.empty: + dataframe = prediction_dataframe.copy() + for tf in tfs: + base_dataframes[tf] = None + for p in pairs: + if p not in corr_dataframes: + corr_dataframes[p] = {} + corr_dataframes[p][tf] = None + else: + dataframe = base_dataframes[self.config["timeframe"]].copy() + + sgi = False + for tf in tfs: + if tf == tfs[-1]: + sgi = True # doing this last allows user to use all tf raw prices in labels + dataframe = strategy.populate_any_indicators( + pair, + dataframe.copy(), + tf, + informative=base_dataframes[tf], + set_generalized_indicators=sgi + ) + if pairs: + for i in pairs: + if pair in i: + continue # dont repeat anything from whitelist + dataframe = strategy.populate_any_indicators( + i, + dataframe.copy(), + tf, + informative=corr_dataframes[i][tf] + ) + + self.get_unique_classes_from_labels(dataframe) + + return dataframe + + def fit_labels(self) -> None: + """ + Fit the labels with a gaussian distribution + """ + import scipy as spy + + self.data["labels_mean"], self.data["labels_std"] = {}, {} + for label in self.data_dictionary["train_labels"].columns: + if self.data_dictionary["train_labels"][label].dtype == object: + continue + f = spy.stats.norm.fit(self.data_dictionary["train_labels"][label]) + self.data["labels_mean"][label], self.data["labels_std"][label] = f[0], f[1] + + # incase targets are classifications + for label in self.unique_class_list: + self.data["labels_mean"][label], self.data["labels_std"][label] = 0, 0 + + return + + def remove_features_from_df(self, dataframe: DataFrame) -> DataFrame: + """ + Remove the features from the dataframe before returning it to strategy. This keeps it + compact for Frequi purposes. + """ + to_keep = [ + col for col in dataframe.columns if not col.startswith("%") or col.startswith("%%") + ] + return dataframe[to_keep] + + def get_unique_classes_from_labels(self, dataframe: DataFrame) -> None: + + self.find_features(dataframe) + + for key in self.label_list: + if dataframe[key].dtype == object: + self.unique_classes[key] = dataframe[key].dropna().unique() + + if self.unique_classes: + for label in self.unique_classes: + self.unique_class_list += list(self.unique_classes[label]) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py new file mode 100644 index 000000000..4106f24e0 --- /dev/null +++ b/freqtrade/freqai/freqai_interface.py @@ -0,0 +1,684 @@ +# import contextlib +import datetime +import logging +import shutil +import threading +import time +from abc import ABC, abstractmethod +from pathlib import Path +from threading import Lock +from typing import Any, Dict, Tuple + +import numpy as np +import pandas as pd +from numpy.typing import NDArray +from pandas import DataFrame + +from freqtrade.configuration import TimeRange +from freqtrade.enums import RunMode +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_seconds +from freqtrade.freqai.data_drawer import FreqaiDataDrawer +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.strategy.interface import IStrategy + + +pd.options.mode.chained_assignment = None +logger = logging.getLogger(__name__) + + +def threaded(fn): + def wrapper(*args, **kwargs): + threading.Thread(target=fn, args=args, kwargs=kwargs).start() + + return wrapper + + +class IFreqaiModel(ABC): + """ + Class containing all tools for training and prediction in the strategy. + Base*PredictionModels inherit from this class. + + Record of contribution: + FreqAI was developed by a group of individuals who all contributed specific skillsets to the + project. + + Conception and software development: + Robert Caulk @robcaulk + + Theoretical brainstorming: + Elin Törnquist @th0rntwig + + Code review, software architecture brainstorming: + @xmatthias + + Beta testing and bug reporting: + @bloodhunter4rc, Salah Lamkadem @ikonx, @ken11o2, @longyu, @paranoidandy, @smidelis, @smarm + Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert + """ + + def __init__(self, config: Dict[str, Any]) -> None: + + self.config = config + self.assert_config(self.config) + self.freqai_info: Dict[str, Any] = config["freqai"] + self.data_split_parameters: Dict[str, Any] = config.get("freqai", {}).get( + "data_split_parameters", {}) + self.model_training_parameters: Dict[str, Any] = config.get("freqai", {}).get( + "model_training_parameters", {}) + self.feature_parameters = config.get("freqai", {}).get("feature_parameters") + self.retrain = False + self.first = True + self.set_full_path() + self.follow_mode: bool = self.freqai_info.get("follow_mode", False) + self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) + self.identifier: str = self.freqai_info.get("identifier", "no_id_provided") + self.scanning = False + self.keras: bool = self.freqai_info.get("keras", False) + if self.keras and self.freqai_info.get("feature_parameters", {}).get("DI_threshold", 0): + self.freqai_info["feature_parameters"]["DI_threshold"] = 0 + logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") + self.CONV_WIDTH = self.freqai_info.get("conv_width", 2) + self.pair_it = 0 + self.pair_it_train = 0 + self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist")) + self.last_trade_database_summary: DataFrame = {} + self.current_trade_database_summary: DataFrame = {} + self.analysis_lock = Lock() + self.inference_time: float = 0 + self.train_time: float = 0 + self.begin_time: float = 0 + self.begin_time_train: float = 0 + self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe']) + + def assert_config(self, config: Dict[str, Any]) -> None: + + if not config.get("freqai", {}): + raise OperationalException("No freqai parameters found in configuration file.") + + def start(self, dataframe: DataFrame, metadata: dict, strategy: IStrategy) -> DataFrame: + """ + Entry point to the FreqaiModel from a specific pair, it will train a new model if + necessary before making the prediction. + + :param dataframe: Full dataframe coming from strategy - it contains entire + backtesting timerange + additional historical data necessary to train + the model. + :param metadata: pair metadata coming from strategy. + :param strategy: Strategy to train on + """ + + self.live = strategy.dp.runmode in (RunMode.DRY_RUN, RunMode.LIVE) + self.dd.set_pair_dict_info(metadata) + + if self.live: + self.inference_timer('start') + self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) + dk = self.start_live(dataframe, metadata, strategy, self.dk) + + # For backtesting, each pair enters and then gets trained for each window along the + # sliding window defined by "train_period_days" (training window) and "live_retrain_hours" + # (backtest window, i.e. window immediately following the training window). + # FreqAI slides the window and sequentially builds the backtesting results before returning + # the concatenated results for the full backtesting period back to the strategy. + elif not self.follow_mode: + self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) + logger.info(f"Training {len(self.dk.training_timeranges)} timeranges") + with self.analysis_lock: + dataframe = self.dk.use_strategy_to_populate_indicators( + strategy, prediction_dataframe=dataframe, pair=metadata["pair"] + ) + dk = self.start_backtesting(dataframe, metadata, self.dk) + + dataframe = dk.remove_features_from_df(dk.return_dataframe) + self.clean_up() + if self.live: + self.inference_timer('stop') + return dataframe + + def clean_up(self): + """ + Objects that should be handled by GC already between coins, but + are explicitly shown here to help demonstrate the non-persistence of these + objects. + """ + self.model = None + self.dk = None + + @threaded + def start_scanning(self, strategy: IStrategy) -> None: + """ + Function designed to constantly scan pairs for retraining on a separate thread (intracandle) + to improve model youth. This function is agnostic to data preparation/collection/storage, + it simply trains on what ever data is available in the self.dd. + :param strategy: IStrategy = The user defined strategy class + """ + while 1: + time.sleep(1) + for pair in self.config.get("exchange", {}).get("pair_whitelist"): + + (_, trained_timestamp, _) = self.dd.get_pair_dict_info(pair) + + if self.dd.pair_dict[pair]["priority"] != 1: + continue + dk = FreqaiDataKitchen(self.config, self.live, pair) + dk.set_paths(pair, trained_timestamp) + ( + retrain, + new_trained_timerange, + data_load_timerange, + ) = dk.check_if_new_training_required(trained_timestamp) + dk.set_paths(pair, new_trained_timerange.stopts) + + if retrain: + self.train_timer('start') + self.train_model_in_series( + new_trained_timerange, pair, strategy, dk, data_load_timerange + ) + self.train_timer('stop') + + self.dd.save_historic_predictions_to_disk() + + def start_backtesting( + self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen + ) -> FreqaiDataKitchen: + """ + The main broad execution for backtesting. For backtesting, each pair enters and then gets + trained for each window along the sliding window defined by "train_period_days" + (training window) and "backtest_period_days" (backtest window, i.e. window immediately + following the training window). FreqAI slides the window and sequentially builds + the backtesting results before returning the concatenated results for the full + backtesting period back to the strategy. + :param dataframe: DataFrame = strategy passed dataframe + :param metadata: Dict = pair metadata + :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only + :return: + FreqaiDataKitchen = Data management/analysis tool associated to present pair only + """ + + self.pair_it += 1 + train_it = 0 + # Loop enforcing the sliding window training/backtesting paradigm + # tr_train is the training time range e.g. 1 historical month + # tr_backtest is the backtesting time range e.g. the week directly + # following tr_train. Both of these windows slide through the + # entire backtest + for tr_train, tr_backtest in zip(dk.training_timeranges, dk.backtesting_timeranges): + (_, _, _) = self.dd.get_pair_dict_info(metadata["pair"]) + train_it += 1 + total_trains = len(dk.backtesting_timeranges) + self.training_timerange = tr_train + dataframe_train = dk.slice_dataframe(tr_train, dataframe) + dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe) + + trained_timestamp = tr_train + tr_train_startts_str = datetime.datetime.utcfromtimestamp(tr_train.startts).strftime( + "%Y-%m-%d %H:%M:%S" + ) + tr_train_stopts_str = datetime.datetime.utcfromtimestamp(tr_train.stopts).strftime( + "%Y-%m-%d %H:%M:%S" + ) + logger.info( + f"Training {metadata['pair']}, {self.pair_it}/{self.total_pairs} pairs" + f" from {tr_train_startts_str} to {tr_train_stopts_str}, {train_it}/{total_trains} " + "trains" + ) + + dk.data_path = Path( + dk.full_path + / + f"sub-train-{metadata['pair'].split('/')[0]}_{int(trained_timestamp.stopts)}" + ) + if not self.model_exists( + metadata["pair"], dk, trained_timestamp=int(trained_timestamp.stopts) + ): + dk.find_features(dataframe_train) + self.model = self.train(dataframe_train, metadata["pair"], dk) + self.dd.pair_dict[metadata["pair"]]["trained_timestamp"] = int( + trained_timestamp.stopts) + dk.set_new_model_names(metadata["pair"], trained_timestamp) + self.dd.save_data(self.model, metadata["pair"], dk) + else: + self.model = self.dd.load_data(metadata["pair"], dk) + + self.check_if_feature_list_matches_strategy(dataframe_train, dk) + + pred_df, do_preds = self.predict(dataframe_backtest, dk) + + dk.append_predictions(pred_df, do_preds) + + dk.fill_predictions(dataframe) + + return dk + + def start_live( + self, dataframe: DataFrame, metadata: dict, strategy: IStrategy, dk: FreqaiDataKitchen + ) -> FreqaiDataKitchen: + """ + The main broad execution for dry/live. This function will check if a retraining should be + performed, and if so, retrain and reset the model. + :param dataframe: DataFrame = strategy passed dataframe + :param metadata: Dict = pair metadata + :param strategy: IStrategy = currently employed strategy + dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only + :returns: + dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only + """ + + # update follower + if self.follow_mode: + self.dd.update_follower_metadata() + + # get the model metadata associated with the current pair + (_, trained_timestamp, return_null_array) = self.dd.get_pair_dict_info(metadata["pair"]) + + # if the metadata doesn't exist, the follower returns null arrays to strategy + if self.follow_mode and return_null_array: + logger.info("Returning null array from follower to strategy") + self.dd.return_null_values_to_strategy(dataframe, dk) + return dk + + # append the historic data once per round + if self.dd.historic_data: + self.dd.update_historic_data(strategy, dk) + logger.debug(f'Updating historic data on pair {metadata["pair"]}') + + if not self.follow_mode: + + (_, new_trained_timerange, data_load_timerange) = dk.check_if_new_training_required( + trained_timestamp + ) + dk.set_paths(metadata["pair"], new_trained_timerange.stopts) + + # download candle history if it is not already in memory + if not self.dd.historic_data: + logger.info( + "Downloading all training data for all pairs in whitelist and " + "corr_pairlist, this may take a while if you do not have the " + "data saved" + ) + dk.download_all_data_for_training(data_load_timerange, strategy.dp) + self.dd.load_all_pair_histories(data_load_timerange, dk) + + if not self.scanning: + self.scanning = True + self.start_scanning(strategy) + + elif self.follow_mode: + dk.set_paths(metadata["pair"], trained_timestamp) + logger.info( + "FreqAI instance set to follow_mode, finding existing pair " + f"using { self.identifier }" + ) + + # load the model and associated data into the data kitchen + self.model = self.dd.load_data(metadata["pair"], dk) + + with self.analysis_lock: + dataframe = self.dk.use_strategy_to_populate_indicators( + strategy, prediction_dataframe=dataframe, pair=metadata["pair"] + ) + + if not self.model: + logger.warning( + f"No model ready for {metadata['pair']}, returning null values to strategy." + ) + self.dd.return_null_values_to_strategy(dataframe, dk) + return dk + + # ensure user is feeding the correct indicators to the model + self.check_if_feature_list_matches_strategy(dataframe, dk) + + self.build_strategy_return_arrays(dataframe, dk, metadata["pair"], trained_timestamp) + + return dk + + def build_strategy_return_arrays( + self, dataframe: DataFrame, dk: FreqaiDataKitchen, pair: str, trained_timestamp: int + ) -> None: + + # hold the historical predictions in memory so we are sending back + # correct array to strategy + + if pair not in self.dd.model_return_values: + # first predictions are made on entire historical candle set coming from strategy. This + # allows FreqUI to show full return values. + pred_df, do_preds = self.predict(dataframe, dk) + if pair not in self.dd.historic_predictions: + self.set_initial_historic_predictions(pred_df, dk, pair) + self.dd.set_initial_return_values(pair, pred_df) + + dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe) + return + elif self.dk.check_if_model_expired(trained_timestamp): + pred_df = DataFrame(np.zeros((2, len(dk.label_list))), columns=dk.label_list) + do_preds = np.ones(2, dtype=np.int_) * 2 + dk.DI_values = np.zeros(2) + logger.warning( + f"Model expired for {pair}, returning null values to strategy. Strategy " + "construction should take care to consider this event with " + "prediction == 0 and do_predict == 2" + ) + else: + # remaining predictions are made only on the most recent candles for performance and + # historical accuracy reasons. + pred_df, do_preds = self.predict(dataframe.iloc[-self.CONV_WIDTH:], dk, first=False) + + if self.freqai_info.get('fit_live_predictions_candles', 0) and self.live: + self.fit_live_predictions(dk, pair) + self.dd.append_model_predictions(pair, pred_df, do_preds, dk, len(dataframe)) + dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe) + + return + + def check_if_feature_list_matches_strategy( + self, dataframe: DataFrame, dk: FreqaiDataKitchen + ) -> None: + """ + Ensure user is passing the proper feature set if they are reusing an `identifier` pointing + to a folder holding existing models. + :param dataframe: DataFrame = strategy provided dataframe + :param dk: FreqaiDataKitchen = non-persistent data container/analyzer for + current coin/bot loop + """ + dk.find_features(dataframe) + if "training_features_list_raw" in dk.data: + feature_list = dk.data["training_features_list_raw"] + else: + feature_list = dk.training_features_list + if dk.training_features_list != feature_list: + raise OperationalException( + "Trying to access pretrained model with `identifier` " + "but found different features furnished by current strategy." + "Change `identifier` to train from scratch, or ensure the" + "strategy is furnishing the same features as the pretrained" + "model" + ) + + def data_cleaning_train(self, dk: FreqaiDataKitchen) -> None: + """ + Base data cleaning method for train + Any function inside this method should drop training data points from the filtered_dataframe + based on user decided logic. See FreqaiDataKitchen::use_SVM_to_remove_outliers() for an + example of how outlier data points are dropped from the dataframe used for training. + """ + + if self.freqai_info["feature_parameters"].get( + "principal_component_analysis", False + ): + dk.principal_component_analysis() + + if self.freqai_info["feature_parameters"].get("use_SVM_to_remove_outliers", False): + dk.use_SVM_to_remove_outliers(predict=False) + + if self.freqai_info["feature_parameters"].get("DI_threshold", 0): + dk.data["avg_mean_dist"] = dk.compute_distances() + + if self.freqai_info["feature_parameters"].get("use_DBSCAN_to_remove_outliers", False): + if dk.pair in self.dd.old_DBSCAN_eps: + eps = self.dd.old_DBSCAN_eps[dk.pair] + else: + eps = None + dk.use_DBSCAN_to_remove_outliers(predict=False, eps=eps) + self.dd.old_DBSCAN_eps[dk.pair] = dk.data['DBSCAN_eps'] + + def data_cleaning_predict(self, dk: FreqaiDataKitchen, dataframe: DataFrame) -> None: + """ + Base data cleaning method for predict. + These functions each modify dk.do_predict, which is a dataframe with equal length + to the number of candles coming from and returning to the strategy. Inside do_predict, + 1 allows prediction and < 0 signals to the strategy that the model is not confident in + the prediction. + See FreqaiDataKitchen::remove_outliers() for an example + of how the do_predict vector is modified. do_predict is ultimately passed back to strategy + for buy signals. + """ + if self.freqai_info["feature_parameters"].get( + "principal_component_analysis", False + ): + dk.pca_transform(dataframe) + + if self.freqai_info["feature_parameters"].get("use_SVM_to_remove_outliers", False): + dk.use_SVM_to_remove_outliers(predict=True) + + if self.freqai_info["feature_parameters"].get("DI_threshold", 0): + dk.check_if_pred_in_training_spaces() + + if self.freqai_info["feature_parameters"].get("use_DBSCAN_to_remove_outliers", False): + dk.use_DBSCAN_to_remove_outliers(predict=True) + + def model_exists( + self, + pair: str, + dk: FreqaiDataKitchen, + trained_timestamp: int = None, + model_filename: str = "", + scanning: bool = False, + ) -> bool: + """ + Given a pair and path, check if a model already exists + :param pair: pair e.g. BTC/USD + :param path: path to model + :return: + :boolean: whether the model file exists or not. + """ + coin, _ = pair.split("/") + + if not self.live: + dk.model_filename = model_filename = f"cb_{coin.lower()}_{trained_timestamp}" + + path_to_modelfile = Path(dk.data_path / f"{model_filename}_model.joblib") + file_exists = path_to_modelfile.is_file() + if file_exists and not scanning: + logger.info("Found model at %s", dk.data_path / dk.model_filename) + elif not scanning: + logger.info("Could not find model at %s", dk.data_path / dk.model_filename) + return file_exists + + def set_full_path(self) -> None: + self.full_path = Path( + self.config["user_data_dir"] / "models" / f"{self.freqai_info['identifier']}" + ) + self.full_path.mkdir(parents=True, exist_ok=True) + shutil.copy( + self.config["config_files"][0], + Path(self.full_path, Path(self.config["config_files"][0]).name), + ) + + def train_model_in_series( + self, + new_trained_timerange: TimeRange, + pair: str, + strategy: IStrategy, + dk: FreqaiDataKitchen, + data_load_timerange: TimeRange, + ): + """ + Retrieve data and train model. + :param new_trained_timerange: TimeRange = the timerange to train the model on + :param metadata: dict = strategy provided metadata + :param strategy: IStrategy = user defined strategy object + :param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop + :param data_load_timerange: TimeRange = the amount of data to be loaded + for populate_any_indicators + (larger than new_trained_timerange so that + new_trained_timerange does not contain any NaNs) + """ + + corr_dataframes, base_dataframes = self.dd.get_base_and_corr_dataframes( + data_load_timerange, pair, dk + ) + + with self.analysis_lock: + unfiltered_dataframe = dk.use_strategy_to_populate_indicators( + strategy, corr_dataframes, base_dataframes, pair + ) + + unfiltered_dataframe = dk.slice_dataframe(new_trained_timerange, unfiltered_dataframe) + + # find the features indicated by strategy and store in datakitchen + dk.find_features(unfiltered_dataframe) + + model = self.train(unfiltered_dataframe, pair, dk) + + self.dd.pair_dict[pair]["trained_timestamp"] = new_trained_timerange.stopts + dk.set_new_model_names(pair, new_trained_timerange) + self.dd.pair_dict[pair]["first"] = False + if self.dd.pair_dict[pair]["priority"] == 1 and self.scanning: + self.dd.pair_to_end_of_training_queue(pair) + self.dd.save_data(model, pair, dk) + + if self.freqai_info.get("purge_old_models", False): + self.dd.purge_old_models() + + def set_initial_historic_predictions( + self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str + ) -> None: + """ + This function is called only if the datadrawer failed to load an + existing set of historic predictions. In this case, it builds + the structure and sets fake predictions off the first training + data. After that, FreqAI will append new real predictions to the + set of historic predictions. + + These values are used to generate live statistics which can be used + in the strategy for adaptive values. E.g. &*_mean/std are quantities + that can computed based on live predictions from the set of historical + predictions. Those values can be used in the user strategy to better + assess prediction rarity, and thus wait for probabilistically favorable + entries relative to the live historical predictions. + + If the user reuses an identifier on a subsequent instance, + this function will not be called. In that case, "real" predictions + will be appended to the loaded set of historic predictions. + :param: df: DataFrame = the dataframe containing the training feature data + :param: model: Any = A model which was `fit` using a common library such as + catboost or lightgbm + :param: dk: FreqaiDataKitchen = object containing methods for data analysis + :param: pair: str = current pair + """ + + self.dd.historic_predictions[pair] = pred_df + hist_preds_df = self.dd.historic_predictions[pair] + + for label in hist_preds_df.columns: + if hist_preds_df[label].dtype == object: + continue + hist_preds_df[f'{label}_mean'] = 0 + hist_preds_df[f'{label}_std'] = 0 + + hist_preds_df['do_predict'] = 0 + + if self.freqai_info['feature_parameters'].get('DI_threshold', 0) > 0: + hist_preds_df['DI_values'] = 0 + + for return_str in dk.data['extra_returns_per_train']: + hist_preds_df[return_str] = 0 + + # # for keras type models, the conv_window needs to be prepended so + # # viewing is correct in frequi + if self.freqai_info.get('keras', False): + n_lost_points = self.freqai_info.get('conv_width', 2) + zeros_df = DataFrame(np.zeros((n_lost_points, len(hist_preds_df.columns))), + columns=hist_preds_df.columns) + self.dd.historic_predictions[pair] = pd.concat( + [zeros_df, hist_preds_df], axis=0, ignore_index=True) + + def fit_live_predictions(self, dk: FreqaiDataKitchen, pair: str) -> None: + """ + Fit the labels with a gaussian distribution + """ + import scipy as spy + + # add classes from classifier label types if used + full_labels = dk.label_list + dk.unique_class_list + + num_candles = self.freqai_info.get("fit_live_predictions_candles", 100) + dk.data["labels_mean"], dk.data["labels_std"] = {}, {} + for label in full_labels: + if self.dd.historic_predictions[dk.pair][label].dtype == object: + continue + f = spy.stats.norm.fit(self.dd.historic_predictions[dk.pair][label].tail(num_candles)) + dk.data["labels_mean"][label], dk.data["labels_std"][label] = f[0], f[1] + + return + + def inference_timer(self, do='start'): + """ + Timer designed to track the cumulative time spent in FreqAI for one pass through + the whitelist. This will check if the time spent is more than 1/4 the time + of a single candle, and if so, it will warn the user of degraded performance + """ + if do == 'start': + self.pair_it += 1 + self.begin_time = time.time() + elif do == 'stop': + end = time.time() + self.inference_time += (end - self.begin_time) + if self.pair_it == self.total_pairs: + logger.info( + f'Total time spent inferencing pairlist {self.inference_time:.2f} seconds') + if self.inference_time > 0.25 * self.base_tf_seconds: + logger.warning('Inference took over 25/% of the candle time. Reduce pairlist to' + ' avoid blinding open trades and degrading performance.') + self.pair_it = 0 + self.inference_time = 0 + return + + def train_timer(self, do='start'): + """ + Timer designed to track the cumulative time spent training the full pairlist in + FreqAI. + """ + if do == 'start': + self.pair_it_train += 1 + self.begin_time_train = time.time() + elif do == 'stop': + end = time.time() + self.train_time += (end - self.begin_time_train) + if self.pair_it_train == self.total_pairs: + logger.info( + f'Total time spent training pairlist {self.train_time:.2f} seconds') + self.pair_it_train = 0 + self.train_time = 0 + return + + # Following methods which are overridden by user made prediction models. + # See freqai/prediction_models/CatboostPredictionModel.py for an example. + + @abstractmethod + def train(self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen) -> Any: + """ + Filter the training data and train a model to it. Train makes heavy use of the datahandler + for storing, saving, loading, and analyzing the data. + :param unfiltered_dataframe: Full dataframe for the current training period + :param metadata: pair metadata from strategy. + :return: Trained model which can be used to inference (self.predict) + """ + + @abstractmethod + def fit(self, data_dictionary: Dict[str, Any]) -> Any: + """ + Most regressors use the same function names and arguments e.g. user + can drop in LGBMRegressor in place of CatBoostRegressor and all data + management will be properly handled by Freqai. + :param data_dictionary: Dict = the dictionary constructed by DataHandler to hold + all the training and test data/labels. + """ + + return + + @abstractmethod + def predict( + self, dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = True + ) -> Tuple[DataFrame, NDArray[np.int_]]: + """ + Filter the prediction features data and predict with it. + :param unfiltered_dataframe: Full dataframe for the current backtest period. + :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only + :param first: boolean = whether this is the first prediction or not. + :return: + :predictions: np.array of predictions + :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove + data (NaNs) or felt uncertain about data (i.e. SVM and/or DI index) + """ diff --git a/freqtrade/freqai/prediction_models/BaseClassifierModel.py b/freqtrade/freqai/prediction_models/BaseClassifierModel.py new file mode 100644 index 000000000..2edbf3b51 --- /dev/null +++ b/freqtrade/freqai/prediction_models/BaseClassifierModel.py @@ -0,0 +1,99 @@ +import logging +from typing import Any, Tuple + +import numpy as np +import numpy.typing as npt +import pandas as pd +from pandas import DataFrame + +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.freqai.freqai_interface import IFreqaiModel + + +logger = logging.getLogger(__name__) + + +class BaseClassifierModel(IFreqaiModel): + """ + Base class for regression type models (e.g. Catboost, LightGBM, XGboost etc.). + User *must* inherit from this class and set fit() and predict(). See example scripts + such as prediction_models/CatboostPredictionModel.py for guidance. + """ + + def train( + self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen + ) -> Any: + """ + Filter the training data and train a model to it. Train makes heavy use of the datakitchen + for storing, saving, loading, and analyzing the data. + :param unfiltered_dataframe: Full dataframe for the current training period + :param metadata: pair metadata from strategy. + :return: + :model: Trained model which can be used to inference (self.predict) + """ + + logger.info("-------------------- Starting training " f"{pair} --------------------") + + # filter the features requested by user in the configuration file and elegantly handle NaNs + features_filtered, labels_filtered = dk.filter_features( + unfiltered_dataframe, + dk.training_features_list, + dk.label_list, + training_filter=True, + ) + + start_date = unfiltered_dataframe["date"].iloc[0].strftime("%Y-%m-%d") + end_date = unfiltered_dataframe["date"].iloc[-1].strftime("%Y-%m-%d") + logger.info(f"-------------------- Training on data from {start_date} to " + f"{end_date}--------------------") + # split data into train/test data. + data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered) + if not self.freqai_info.get('fit_live_predictions', 0) or not self.live: + dk.fit_labels() + # normalize all data based on train_dataset only + data_dictionary = dk.normalize_data(data_dictionary) + + # optional additional data cleaning/analysis + self.data_cleaning_train(dk) + + logger.info( + f'Training model on {len(dk.data_dictionary["train_features"].columns)}' " features" + ) + logger.info(f'Training model on {len(data_dictionary["train_features"])} data points') + + model = self.fit(data_dictionary) + + logger.info(f"--------------------done training {pair}--------------------") + + return model + + def predict( + self, unfiltered_dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = False + ) -> Tuple[DataFrame, npt.NDArray[np.int_]]: + """ + Filter the prediction features data and predict with it. + :param: unfiltered_dataframe: Full dataframe for the current backtest period. + :return: + :pred_df: dataframe containing the predictions + :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove + data (NaNs) or felt uncertain about data (PCA and DI index) + """ + + dk.find_features(unfiltered_dataframe) + filtered_dataframe, _ = dk.filter_features( + unfiltered_dataframe, dk.training_features_list, training_filter=False + ) + filtered_dataframe = dk.normalize_data_from_metadata(filtered_dataframe) + dk.data_dictionary["prediction_features"] = filtered_dataframe + + self.data_cleaning_predict(dk, filtered_dataframe) + + predictions = self.model.predict(dk.data_dictionary["prediction_features"]) + pred_df = DataFrame(predictions, columns=dk.label_list) + + predictions_prob = self.model.predict_proba(dk.data_dictionary["prediction_features"]) + pred_df_prob = DataFrame(predictions_prob, columns=self.model.classes_) + + pred_df = pd.concat([pred_df, pred_df_prob], axis=1) + + return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/prediction_models/BaseRegressionModel.py b/freqtrade/freqai/prediction_models/BaseRegressionModel.py new file mode 100644 index 000000000..2ef175a2e --- /dev/null +++ b/freqtrade/freqai/prediction_models/BaseRegressionModel.py @@ -0,0 +1,96 @@ +import logging +from typing import Any, Tuple + +import numpy as np +import numpy.typing as npt +from pandas import DataFrame + +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.freqai.freqai_interface import IFreqaiModel + + +logger = logging.getLogger(__name__) + + +class BaseRegressionModel(IFreqaiModel): + """ + Base class for regression type models (e.g. Catboost, LightGBM, XGboost etc.). + User *must* inherit from this class and set fit() and predict(). See example scripts + such as prediction_models/CatboostPredictionModel.py for guidance. + """ + + def train( + self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen + ) -> Any: + """ + Filter the training data and train a model to it. Train makes heavy use of the datakitchen + for storing, saving, loading, and analyzing the data. + :param unfiltered_dataframe: Full dataframe for the current training period + :param metadata: pair metadata from strategy. + :return: + :model: Trained model which can be used to inference (self.predict) + """ + + logger.info("-------------------- Starting training " f"{pair} --------------------") + + # filter the features requested by user in the configuration file and elegantly handle NaNs + features_filtered, labels_filtered = dk.filter_features( + unfiltered_dataframe, + dk.training_features_list, + dk.label_list, + training_filter=True, + ) + + start_date = unfiltered_dataframe["date"].iloc[0].strftime("%Y-%m-%d") + end_date = unfiltered_dataframe["date"].iloc[-1].strftime("%Y-%m-%d") + logger.info(f"-------------------- Training on data from {start_date} to " + f"{end_date}--------------------") + # split data into train/test data. + data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered) + if not self.freqai_info.get('fit_live_predictions', 0) or not self.live: + dk.fit_labels() + # normalize all data based on train_dataset only + data_dictionary = dk.normalize_data(data_dictionary) + + # optional additional data cleaning/analysis + self.data_cleaning_train(dk) + + logger.info( + f'Training model on {len(dk.data_dictionary["train_features"].columns)}' " features" + ) + logger.info(f'Training model on {len(data_dictionary["train_features"])} data points') + + model = self.fit(data_dictionary) + + logger.info(f"--------------------done training {pair}--------------------") + + return model + + def predict( + self, unfiltered_dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = False + ) -> Tuple[DataFrame, npt.NDArray[np.int_]]: + """ + Filter the prediction features data and predict with it. + :param: unfiltered_dataframe: Full dataframe for the current backtest period. + :return: + :pred_df: dataframe containing the predictions + :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove + data (NaNs) or felt uncertain about data (PCA and DI index) + """ + + dk.find_features(unfiltered_dataframe) + filtered_dataframe, _ = dk.filter_features( + unfiltered_dataframe, dk.training_features_list, training_filter=False + ) + filtered_dataframe = dk.normalize_data_from_metadata(filtered_dataframe) + dk.data_dictionary["prediction_features"] = filtered_dataframe + + # optional additional data cleaning/analysis + self.data_cleaning_predict(dk, filtered_dataframe) + + predictions = self.model.predict(dk.data_dictionary["prediction_features"]) + pred_df = DataFrame(predictions, columns=dk.label_list) + + pred_df = dk.denormalize_labels_from_metadata(pred_df) + + return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py b/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py new file mode 100644 index 000000000..04eff045f --- /dev/null +++ b/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py @@ -0,0 +1,64 @@ +import logging +from typing import Any + +from pandas import DataFrame + +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.freqai.freqai_interface import IFreqaiModel + + +logger = logging.getLogger(__name__) + + +class BaseTensorFlowModel(IFreqaiModel): + """ + Base class for TensorFlow type models. + User *must* inherit from this class and set fit() and predict(). + """ + + def train( + self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen + ) -> Any: + """ + Filter the training data and train a model to it. Train makes heavy use of the datakitchen + for storing, saving, loading, and analyzing the data. + :param unfiltered_dataframe: Full dataframe for the current training period + :param metadata: pair metadata from strategy. + :return: + :model: Trained model which can be used to inference (self.predict) + """ + + logger.info("-------------------- Starting training " f"{pair} --------------------") + + # filter the features requested by user in the configuration file and elegantly handle NaNs + features_filtered, labels_filtered = dk.filter_features( + unfiltered_dataframe, + dk.training_features_list, + dk.label_list, + training_filter=True, + ) + + start_date = unfiltered_dataframe["date"].iloc[0].strftime("%Y-%m-%d") + end_date = unfiltered_dataframe["date"].iloc[-1].strftime("%Y-%m-%d") + logger.info(f"-------------------- Training on data from {start_date} to " + f"{end_date}--------------------") + # split data into train/test data. + data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered) + if not self.freqai_info.get('fit_live_predictions', 0) or not self.live: + dk.fit_labels() + # normalize all data based on train_dataset only + data_dictionary = dk.normalize_data(data_dictionary) + + # optional additional data cleaning/analysis + self.data_cleaning_train(dk) + + logger.info( + f'Training model on {len(dk.data_dictionary["train_features"].columns)}' " features" + ) + logger.info(f'Training model on {len(data_dictionary["train_features"])} data points') + + model = self.fit(data_dictionary) + + logger.info(f"--------------------done training {pair}--------------------") + + return model diff --git a/freqtrade/freqai/prediction_models/CatboostClassifier.py b/freqtrade/freqai/prediction_models/CatboostClassifier.py new file mode 100644 index 000000000..b88b28b25 --- /dev/null +++ b/freqtrade/freqai/prediction_models/CatboostClassifier.py @@ -0,0 +1,41 @@ +import logging +from typing import Any, Dict + +from catboost import CatBoostClassifier, Pool + +from freqtrade.freqai.prediction_models.BaseClassifierModel import BaseClassifierModel + + +logger = logging.getLogger(__name__) + + +class CatboostClassifier(BaseClassifierModel): + """ + User created prediction model. The class needs to override three necessary + functions, predict(), train(), fit(). The class inherits ModelHandler which + has its own DataHandler where data is held, saved, loaded, and managed. + """ + + def fit(self, data_dictionary: Dict) -> Any: + """ + User sets up the training and test data to fit their desired model here + :params: + :data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. + """ + + train_data = Pool( + data=data_dictionary["train_features"], + label=data_dictionary["train_labels"], + weight=data_dictionary["train_weights"], + ) + + cbr = CatBoostClassifier( + allow_writing_files=False, + loss_function='MultiClass', + **self.model_training_parameters, + ) + + cbr.fit(train_data) + + return cbr diff --git a/freqtrade/freqai/prediction_models/CatboostRegressor.py b/freqtrade/freqai/prediction_models/CatboostRegressor.py new file mode 100644 index 000000000..d93569c91 --- /dev/null +++ b/freqtrade/freqai/prediction_models/CatboostRegressor.py @@ -0,0 +1,53 @@ +import gc +import logging +from typing import Any, Dict + +from catboost import CatBoostRegressor, Pool + +from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel + + +logger = logging.getLogger(__name__) + + +class CatboostRegressor(BaseRegressionModel): + """ + User created prediction model. The class needs to override three necessary + functions, predict(), train(), fit(). The class inherits ModelHandler which + has its own DataHandler where data is held, saved, loaded, and managed. + """ + + def fit(self, data_dictionary: Dict) -> Any: + """ + User sets up the training and test data to fit their desired model here + :param data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. + """ + + train_data = Pool( + data=data_dictionary["train_features"], + label=data_dictionary["train_labels"], + weight=data_dictionary["train_weights"], + ) + if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0: + test_data = None + else: + test_data = Pool( + data=data_dictionary["test_features"], + label=data_dictionary["test_labels"], + weight=data_dictionary["test_weights"], + ) + + model = CatBoostRegressor( + allow_writing_files=False, + **self.model_training_parameters, + ) + + model.fit(X=train_data, eval_set=test_data) + + # some evidence that catboost pools have memory leaks: + # https://github.com/catboost/catboost/issues/1835 + del train_data, test_data + gc.collect() + + return model diff --git a/freqtrade/freqai/prediction_models/CatboostRegressorMultiTarget.py b/freqtrade/freqai/prediction_models/CatboostRegressorMultiTarget.py new file mode 100644 index 000000000..9894decd1 --- /dev/null +++ b/freqtrade/freqai/prediction_models/CatboostRegressorMultiTarget.py @@ -0,0 +1,44 @@ +import logging +from typing import Any, Dict + +from catboost import CatBoostRegressor # , Pool +from sklearn.multioutput import MultiOutputRegressor + +from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel + + +logger = logging.getLogger(__name__) + + +class CatboostRegressorMultiTarget(BaseRegressionModel): + """ + User created prediction model. The class needs to override three necessary + functions, predict(), train(), fit(). The class inherits ModelHandler which + has its own DataHandler where data is held, saved, loaded, and managed. + """ + + def fit(self, data_dictionary: Dict) -> Any: + """ + User sets up the training and test data to fit their desired model here + :param data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. + """ + + cbr = CatBoostRegressor( + allow_writing_files=False, + **self.model_training_parameters, + ) + + X = data_dictionary["train_features"] + y = data_dictionary["train_labels"] + eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"]) + sample_weight = data_dictionary["train_weights"] + + model = MultiOutputRegressor(estimator=cbr) + model.fit(X=X, y=y, sample_weight=sample_weight) # , eval_set=eval_set) + + if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) != 0: + train_score = model.score(X, y) + test_score = model.score(*eval_set) + logger.info(f"Train score {train_score}, Test score {test_score}") + return model diff --git a/freqtrade/freqai/prediction_models/LightGBMClassifier.py b/freqtrade/freqai/prediction_models/LightGBMClassifier.py new file mode 100644 index 000000000..4ac2c448b --- /dev/null +++ b/freqtrade/freqai/prediction_models/LightGBMClassifier.py @@ -0,0 +1,43 @@ +import logging +from typing import Any, Dict + +from lightgbm import LGBMClassifier + +from freqtrade.freqai.prediction_models.BaseClassifierModel import BaseClassifierModel + + +logger = logging.getLogger(__name__) + + +class LightGBMClassifier(BaseClassifierModel): + """ + User created prediction model. The class needs to override three necessary + functions, predict(), train(), fit(). The class inherits ModelHandler which + has its own DataHandler where data is held, saved, loaded, and managed. + """ + + def fit(self, data_dictionary: Dict) -> Any: + """ + User sets up the training and test data to fit their desired model here + :params: + :data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. + """ + + if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0: + eval_set = None + test_weights = None + else: + eval_set = (data_dictionary["test_features"].to_numpy(), + data_dictionary["test_labels"].to_numpy()[:, 0]) + test_weights = data_dictionary["test_weights"] + X = data_dictionary["train_features"].to_numpy() + y = data_dictionary["train_labels"].to_numpy()[:, 0] + train_weights = data_dictionary["train_weights"] + + model = LGBMClassifier(**self.model_training_parameters) + + model.fit(X=X, y=y, eval_set=eval_set, sample_weight=train_weights, + eval_sample_weight=[test_weights]) + + return model diff --git a/freqtrade/freqai/prediction_models/LightGBMRegressor.py b/freqtrade/freqai/prediction_models/LightGBMRegressor.py new file mode 100644 index 000000000..2431fd2ad --- /dev/null +++ b/freqtrade/freqai/prediction_models/LightGBMRegressor.py @@ -0,0 +1,43 @@ +import logging +from typing import Any, Dict + +from lightgbm import LGBMRegressor + +from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel + + +logger = logging.getLogger(__name__) + + +class LightGBMRegressor(BaseRegressionModel): + """ + User created prediction model. The class needs to override three necessary + functions, predict(), train(), fit(). The class inherits ModelHandler which + has its own DataHandler where data is held, saved, loaded, and managed. + """ + + def fit(self, data_dictionary: Dict) -> Any: + """ + Most regressors use the same function names and arguments e.g. user + can drop in LGBMRegressor in place of CatBoostRegressor and all data + management will be properly handled by Freqai. + :param data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. + """ + + if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0: + eval_set = None + eval_weights = None + else: + eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"]) + eval_weights = data_dictionary["test_weights"] + X = data_dictionary["train_features"] + y = data_dictionary["train_labels"] + train_weights = data_dictionary["train_weights"] + + model = LGBMRegressor(**self.model_training_parameters) + + model.fit(X=X, y=y, eval_set=eval_set, sample_weight=train_weights, + eval_sample_weight=[eval_weights]) + + return model diff --git a/freqtrade/freqai/prediction_models/LightGBMRegressorMultiTarget.py b/freqtrade/freqai/prediction_models/LightGBMRegressorMultiTarget.py new file mode 100644 index 000000000..ecd405369 --- /dev/null +++ b/freqtrade/freqai/prediction_models/LightGBMRegressorMultiTarget.py @@ -0,0 +1,39 @@ +import logging +from typing import Any, Dict + +from lightgbm import LGBMRegressor +from sklearn.multioutput import MultiOutputRegressor + +from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel + + +logger = logging.getLogger(__name__) + + +class LightGBMRegressorMultiTarget(BaseRegressionModel): + """ + User created prediction model. The class needs to override three necessary + functions, predict(), train(), fit(). The class inherits ModelHandler which + has its own DataHandler where data is held, saved, loaded, and managed. + """ + + def fit(self, data_dictionary: Dict) -> Any: + """ + User sets up the training and test data to fit their desired model here + :param data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. + """ + + lgb = LGBMRegressor(**self.model_training_parameters) + + X = data_dictionary["train_features"] + y = data_dictionary["train_labels"] + eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"]) + sample_weight = data_dictionary["train_weights"] + + model = MultiOutputRegressor(estimator=lgb) + model.fit(X=X, y=y, sample_weight=sample_weight) # , eval_set=eval_set) + train_score = model.score(X, y) + test_score = model.score(*eval_set) + logger.info(f"Train score {train_score}, Test score {test_score}") + return model diff --git a/freqtrade/freqai/prediction_models/__init__.py b/freqtrade/freqai/prediction_models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 43608cae7..35ba6bab2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -21,17 +21,17 @@ from freqtrade.enums import (ExitCheckTuple, ExitType, RPCMessageType, RunMode, State, TradingMode) from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) -from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds -from freqtrade.exchange.exchange import timeframe_to_next_date +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.mixins import LoggingMixin -from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db +from freqtrade.persistence import Order, PairLocks, Trade, init_db from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper +from freqtrade.util import FtPrecise from freqtrade.wallets import Wallets @@ -149,7 +149,7 @@ class FreqtradeBot(LoggingMixin): self.check_for_open_trades() self.rpc.cleanup() - cleanup_db() + Trade.commit() self.exchange.close() def startup(self) -> None: @@ -158,6 +158,8 @@ class FreqtradeBot(LoggingMixin): performs startup tasks """ self.rpc.startup_messages(self.config, self.pairlists, self.protections) + # Update older trades with precision and precision mode + self.startup_backpopulate_precision() if not self.edge: # Adjust stoploss if it was changed Trade.stoploss_reinitialization(self.strategy.stoploss) @@ -214,6 +216,7 @@ class FreqtradeBot(LoggingMixin): if self.trading_mode == TradingMode.FUTURES: self._schedule.run_pending() Trade.commit() + self.rpc.process_msg_queue(self.dataprovider._msg_queue) self.last_process = datetime.now(timezone.utc) def process_stopped(self) -> None: @@ -236,7 +239,7 @@ class FreqtradeBot(LoggingMixin): 'status': f"{len(open_trades)} open trades active.\n\n" f"Handle these trades manually on {self.exchange.name}, " - f"or '/start' the bot again and use '/stopbuy' " + f"or '/start' the bot again and use '/stopentry' " f"to handle open trades gracefully. \n" f"{'Note: Trades are simulated (dry run).' if self.config['dry_run'] else ''}", } @@ -267,7 +270,7 @@ class FreqtradeBot(LoggingMixin): Return the number of free open trades slots or 0 if max number of open trades reached """ - open_trades = len(Trade.get_open_trades()) + open_trades = Trade.get_open_trade_count() return max(0, self.config['max_open_trades'] - open_trades) def update_funding_fees(self): @@ -284,6 +287,18 @@ class FreqtradeBot(LoggingMixin): else: return 0.0 + def startup_backpopulate_precision(self): + + trades = Trade.get_trades([Trade.contract_size.is_(None)]) + for trade in trades: + if trade.exchange != self.exchange.id: + continue + trade.precision_mode = self.exchange.precisionMode + trade.amount_precision = self.exchange.get_precision_amount(trade.pair) + trade.price_precision = self.exchange.get_precision_price(trade.pair) + trade.contract_size = self.exchange.get_contract_size(trade.pair) + Trade.commit() + def startup_update_open_orders(self): """ Updates open orders based on order list kept in the database. @@ -403,7 +418,7 @@ class FreqtradeBot(LoggingMixin): whitelist = copy.deepcopy(self.active_pair_whitelist) if not whitelist: - logger.info("Active pair whitelist is empty.") + self.log_once("Active pair whitelist is empty.", logger.info) return trades_created # Remove pairs for currently opened trades from the whitelist for trade in Trade.get_open_trades(): @@ -412,8 +427,8 @@ class FreqtradeBot(LoggingMixin): logger.debug('Ignoring %s in pair whitelist', trade.pair) if not whitelist: - logger.info("No currency pair in active pair whitelist, " - "but checking to exit open trades.") + self.log_once("No currency pair in active pair whitelist, " + "but checking to exit open trades.", logger.info) return trades_created if PairLocks.is_global_lock(side='*'): # This only checks for total locks (both sides). @@ -524,39 +539,61 @@ class FreqtradeBot(LoggingMixin): If the strategy triggers the adjustment, a new order gets issued. Once that completes, the existing trade is modified to match new data. """ - if self.strategy.max_entry_position_adjustment > -1: - count_of_buys = trade.nr_of_successful_entries - if count_of_buys > self.strategy.max_entry_position_adjustment: - logger.debug(f"Max adjustment entries for {trade.pair} has been reached.") - return - else: - logger.debug("Max adjustment entries is set to unlimited.") - current_rate = self.exchange.get_rate( - trade.pair, side='entry', is_short=trade.is_short, refresh=True) - current_profit = trade.calc_profit_ratio(current_rate) + current_entry_rate, current_exit_rate = self.exchange.get_rates( + trade.pair, True, trade.is_short) - min_stake_amount = self.exchange.get_min_pair_stake_amount(trade.pair, - current_rate, - self.strategy.stoploss) - max_stake_amount = self.exchange.get_max_pair_stake_amount(trade.pair, current_rate) + current_entry_profit = trade.calc_profit_ratio(current_entry_rate) + current_exit_profit = trade.calc_profit_ratio(current_exit_rate) + + min_entry_stake = self.exchange.get_min_pair_stake_amount(trade.pair, + current_entry_rate, + self.strategy.stoploss) + min_exit_stake = self.exchange.get_min_pair_stake_amount(trade.pair, + current_exit_rate, + self.strategy.stoploss) + max_entry_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_entry_rate) stake_available = self.wallets.get_available_stake_amount() logger.debug(f"Calling adjust_trade_position for pair {trade.pair}") stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position, default_retval=None)( - trade=trade, current_time=datetime.now(timezone.utc), current_rate=current_rate, - current_profit=current_profit, min_stake=min_stake_amount, - max_stake=min(max_stake_amount, stake_available)) + trade=trade, + current_time=datetime.now(timezone.utc), current_rate=current_entry_rate, + current_profit=current_entry_profit, min_stake=min_entry_stake, + max_stake=min(max_entry_stake, stake_available), + current_entry_rate=current_entry_rate, current_exit_rate=current_exit_rate, + current_entry_profit=current_entry_profit, current_exit_profit=current_exit_profit + ) if stake_amount is not None and stake_amount > 0.0: # We should increase our position - self.execute_entry(trade.pair, stake_amount, price=current_rate, + if self.strategy.max_entry_position_adjustment > -1: + count_of_entries = trade.nr_of_successful_entries + if count_of_entries > self.strategy.max_entry_position_adjustment: + logger.debug(f"Max adjustment entries for {trade.pair} has been reached.") + return + else: + logger.debug("Max adjustment entries is set to unlimited.") + self.execute_entry(trade.pair, stake_amount, price=current_entry_rate, trade=trade, is_short=trade.is_short) if stake_amount is not None and stake_amount < 0.0: # We should decrease our position - # TODO: Selling part of the trade not implemented yet. - logger.error(f"Unable to decrease trade position / sell partially" - f" for pair {trade.pair}, feature not implemented.") + amount = abs(float(FtPrecise(stake_amount) / FtPrecise(current_exit_rate))) + if amount > trade.amount: + # This is currently ineffective as remaining would become < min tradable + # Fixing this would require checking for 0.0 there - + # if we decide that this callback is allowed to "fully exit" + logger.info( + f"Adjusting amount to trade.amount as it is higher. {amount} > {trade.amount}") + amount = trade.amount + + remaining = (trade.amount - amount) * current_exit_rate + if remaining < min_exit_stake: + logger.info(f'Remaining amount of {remaining} would be too small.') + return + + self.execute_trade_exit(trade, current_exit_rate, exit_check=ExitCheckTuple( + exit_type=ExitType.PARTIAL_EXIT), sub_trade_amt=amount) def _check_depth_of_market(self, pair: str, conf: Dict, side: SignalDirection) -> bool: """ @@ -600,7 +637,8 @@ class FreqtradeBot(LoggingMixin): ordertype: Optional[str] = None, enter_tag: Optional[str] = None, trade: Optional[Trade] = None, - order_adjust: bool = False + order_adjust: bool = False, + leverage_: Optional[float] = None, ) -> bool: """ Executes a limit buy for the given pair @@ -616,7 +654,7 @@ class FreqtradeBot(LoggingMixin): pos_adjust = trade is not None enter_limit_requested, stake_amount, leverage = self.get_valid_enter_price_and_stake( - pair, price, stake_amount, trade_side, enter_tag, trade, order_adjust) + pair, price, stake_amount, trade_side, enter_tag, trade, order_adjust, leverage_) if not stake_amount: return False @@ -713,7 +751,11 @@ class FreqtradeBot(LoggingMixin): leverage=leverage, is_short=is_short, trading_mode=self.trading_mode, - funding_fees=funding_fees + funding_fees=funding_fees, + amount_precision=self.exchange.get_precision_amount(pair), + price_precision=self.exchange.get_precision_price(pair), + precision_mode=self.exchange.precisionMode, + contract_size=self.exchange.get_contract_size(pair), ) else: # This is additional buy, we reset fee_open_currency so timeout checking can work @@ -730,7 +772,7 @@ class FreqtradeBot(LoggingMixin): # Updating wallets self.wallets.update() - self._notify_enter(trade, order, order_type) + self._notify_enter(trade, order_obj, order_type, sub_trade=pos_adjust) if pos_adjust: if order_status == 'closed': @@ -739,8 +781,8 @@ class FreqtradeBot(LoggingMixin): else: logger.info(f"DCA order {order_status}, will wait for resolution: {trade}") - # Update fees if order is closed - if order_status == 'closed': + # Update fees if order is non-opened + if order_status in constants.NON_OPEN_EXCHANGE_STATES: self.update_trade_state(trade, order_id, order) return True @@ -763,6 +805,7 @@ class FreqtradeBot(LoggingMixin): entry_tag: Optional[str], trade: Optional[Trade], order_adjust: bool, + leverage_: Optional[float], ) -> Tuple[float, float, float]: if price: @@ -785,16 +828,19 @@ class FreqtradeBot(LoggingMixin): if not enter_limit_requested: raise PricingError('Could not determine entry price.') - if trade is None: + if self.trading_mode != TradingMode.SPOT and trade is None: max_leverage = self.exchange.get_max_leverage(pair, stake_amount) - leverage = strategy_safe_wrapper(self.strategy.leverage, default_retval=1.0)( - pair=pair, - current_time=datetime.now(timezone.utc), - current_rate=enter_limit_requested, - proposed_leverage=1.0, - max_leverage=max_leverage, - side=trade_side, entry_tag=entry_tag, - ) if self.trading_mode != TradingMode.SPOT else 1.0 + if leverage_: + leverage = leverage_ + else: + leverage = strategy_safe_wrapper(self.strategy.leverage, default_retval=1.0)( + pair=pair, + current_time=datetime.now(timezone.utc), + current_rate=enter_limit_requested, + proposed_leverage=1.0, + max_leverage=max_leverage, + side=trade_side, entry_tag=entry_tag, + ) # Cap leverage between 1.0 and max_leverage. leverage = min(max(leverage, 1.0), max_leverage) else: @@ -829,13 +875,14 @@ class FreqtradeBot(LoggingMixin): return enter_limit_requested, stake_amount, leverage - def _notify_enter(self, trade: Trade, order: Dict, order_type: Optional[str] = None, - fill: bool = False) -> None: + def _notify_enter(self, trade: Trade, order: Order, order_type: Optional[str] = None, + fill: bool = False, sub_trade: bool = False) -> None: """ Sends rpc notification when a entry order occurred. """ msg_type = RPCMessageType.ENTRY_FILL if fill else RPCMessageType.ENTRY - open_rate = safe_value_fallback(order, 'average', 'price') + open_rate = order.safe_price + if open_rate is None: open_rate = trade.open_rate @@ -859,15 +906,17 @@ class FreqtradeBot(LoggingMixin): 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency', None), - 'amount': safe_value_fallback(order, 'filled', 'amount') or trade.amount, + 'amount': order.safe_amount_after_fee, 'open_date': trade.open_date or datetime.utcnow(), 'current_rate': current_rate, + 'sub_trade': sub_trade, } # Send the message self.rpc.send_msg(msg) - def _notify_enter_cancel(self, trade: Trade, order_type: str, reason: str) -> None: + def _notify_enter_cancel(self, trade: Trade, order_type: str, reason: str, + sub_trade: bool = False) -> None: """ Sends rpc notification when a entry order cancel occurred. """ @@ -892,6 +941,7 @@ class FreqtradeBot(LoggingMixin): 'open_date': trade.open_date, 'current_rate': current_rate, 'reason': reason, + 'sub_trade': sub_trade, } # Send the message @@ -1015,7 +1065,7 @@ class FreqtradeBot(LoggingMixin): trade.stoploss_order_id = None logger.error(f'Unable to place a stoploss order on exchange. {e}') logger.warning('Exiting the trade forcefully') - self.execute_trade_exit(trade, trade.stop_loss, exit_check=ExitCheckTuple( + self.execute_trade_exit(trade, stop_price, exit_check=ExitCheckTuple( exit_type=ExitType.EMERGENCY_EXIT)) except ExchangeError: @@ -1085,7 +1135,7 @@ class FreqtradeBot(LoggingMixin): if (trade.is_open and stoploss_order and stoploss_order['status'] in ('canceled', 'cancelled')): - if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss): + if self.create_stoploss_order(trade=trade, stop_price=trade.stoploss_or_liquidation): return False else: trade.stoploss_order_id = None @@ -1114,7 +1164,7 @@ class FreqtradeBot(LoggingMixin): :param order: Current on exchange stoploss order :return: None """ - stoploss_norm = self.exchange.price_to_precision(trade.pair, trade.stop_loss) + stoploss_norm = self.exchange.price_to_precision(trade.pair, trade.stoploss_or_liquidation) if self.exchange.stoploss_adjust(stoploss_norm, order, side=trade.exit_side): # we check if the update is necessary @@ -1132,7 +1182,7 @@ class FreqtradeBot(LoggingMixin): f"for pair {trade.pair}") # Create new stoploss order - if not self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss): + if not self.create_stoploss_order(trade=trade, stop_price=stoploss_norm): logger.warning(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") @@ -1365,16 +1415,22 @@ class FreqtradeBot(LoggingMixin): trade.open_order_id = None trade.exit_reason = None cancelled = True + self.wallets.update() else: # TODO: figure out how to handle partially complete sell orders reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] cancelled = False - self.wallets.update() + order_obj = trade.select_order_by_order_id(order['id']) + if not order_obj: + raise DependencyException( + f"Order_obj not found for {order['id']}. This should not have happened.") + + sub_trade = order_obj.amount != trade.amount self._notify_exit_cancel( trade, order_type=self.strategy.order_types['exit'], - reason=reason + reason=reason, order=order_obj, sub_trade=sub_trade ) return cancelled @@ -1415,6 +1471,7 @@ class FreqtradeBot(LoggingMixin): *, exit_tag: Optional[str] = None, ordertype: Optional[str] = None, + sub_trade_amt: float = None, ) -> bool: """ Executes a trade exit for the given trade and limit @@ -1431,15 +1488,10 @@ class FreqtradeBot(LoggingMixin): ) exit_type = 'exit' exit_reason = exit_tag or exit_check.exit_reason - if exit_check.exit_type in (ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS): + if exit_check.exit_type in ( + ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS, ExitType.LIQUIDATION): exit_type = 'stoploss' - # if stoploss is on exchange and we are on dry_run mode, - # we consider the sell price stop price - if (self.config['dry_run'] and exit_type == 'stoploss' - and self.strategy.order_types['stoploss_on_exchange']): - limit = trade.stop_loss - # set custom_exit_price if available proposed_limit_rate = limit current_profit = trade.calc_profit_ratio(limit) @@ -1460,14 +1512,17 @@ class FreqtradeBot(LoggingMixin): # Emergency sells (default to market!) order_type = self.strategy.order_types.get("emergency_exit", "market") - amount = self._safe_exit_amount(trade.pair, trade.amount) + amount = self._safe_exit_amount(trade.pair, sub_trade_amt or trade.amount) time_in_force = self.strategy.order_time_in_force['exit'] - if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( - pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit, - time_in_force=time_in_force, exit_reason=exit_reason, - sell_reason=exit_reason, # sellreason -> compatibility - current_time=datetime.now(timezone.utc)): + if (exit_check.exit_type != ExitType.LIQUIDATION + and not sub_trade_amt + and not strategy_safe_wrapper( + self.strategy.confirm_trade_exit, default_retval=True)( + pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit, + time_in_force=time_in_force, exit_reason=exit_reason, + sell_reason=exit_reason, # sellreason -> compatibility + current_time=datetime.now(timezone.utc))): logger.info(f"User denied exit for {trade.pair}.") return False @@ -1497,11 +1552,12 @@ class FreqtradeBot(LoggingMixin): trade.close_rate_requested = limit trade.exit_reason = exit_reason - # Lock pair for one candle to prevent immediate re-trading - self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), - reason='Auto lock') + if not sub_trade_amt: + # Lock pair for one candle to prevent immediate re-trading + self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), + reason='Auto lock') - self._notify_exit(trade, order_type) + self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj) # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') in ('closed', 'expired'): self.update_trade_state(trade, trade.open_order_id, order) @@ -1509,16 +1565,27 @@ class FreqtradeBot(LoggingMixin): return True - def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False) -> None: + def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False, + sub_trade: bool = False, order: Order = None) -> None: """ Sends rpc notification when a sell occurred. """ - profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested - profit_trade = trade.calc_profit(rate=profit_rate) # Use cached rates here - it was updated seconds ago. current_rate = self.exchange.get_rate( trade.pair, side='exit', is_short=trade.is_short, refresh=False) if not fill else None - profit_ratio = trade.calc_profit_ratio(profit_rate) + + # second condition is for mypy only; order will always be passed during sub trade + if sub_trade and order is not None: + amount = order.safe_filled if fill else order.amount + profit_rate = order.safe_price + + profit = trade.calc_profit(rate=profit_rate, amount=amount, open_rate=trade.open_rate) + profit_ratio = trade.calc_profit_ratio(profit_rate, amount, trade.open_rate) + else: + profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested + profit = trade.calc_profit(rate=profit_rate) + (0.0 if fill else trade.realized_profit) + profit_ratio = trade.calc_profit_ratio(profit_rate) + amount = trade.amount gain = "profit" if profit_ratio > 0 else "loss" msg = { @@ -1532,11 +1599,11 @@ class FreqtradeBot(LoggingMixin): 'gain': gain, 'limit': profit_rate, 'order_type': order_type, - 'amount': trade.amount, + 'amount': amount, 'open_rate': trade.open_rate, - 'close_rate': trade.close_rate, + 'close_rate': profit_rate, 'current_rate': current_rate, - 'profit_amount': profit_trade, + 'profit_amount': profit, 'profit_ratio': profit_ratio, 'buy_tag': trade.enter_tag, 'enter_tag': trade.enter_tag, @@ -1544,19 +1611,18 @@ class FreqtradeBot(LoggingMixin): 'exit_reason': trade.exit_reason, 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.utcnow(), + 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency'), + 'sub_trade': sub_trade, + 'cumulative_profit': trade.realized_profit, } - if 'fiat_display_currency' in self.config: - msg.update({ - 'fiat_currency': self.config['fiat_display_currency'], - }) - # Send the message self.rpc.send_msg(msg) - def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str) -> None: + def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str, + order: Order, sub_trade: bool = False) -> None: """ Sends rpc notification when a sell cancel occurred. """ @@ -1582,7 +1648,7 @@ class FreqtradeBot(LoggingMixin): 'gain': gain, 'limit': profit_rate or 0, 'order_type': order_type, - 'amount': trade.amount, + 'amount': order.safe_amount_after_fee, 'open_rate': trade.open_rate, 'current_rate': current_rate, 'profit_amount': profit_trade, @@ -1596,6 +1662,8 @@ class FreqtradeBot(LoggingMixin): 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency', None), 'reason': reason, + 'sub_trade': sub_trade, + 'stake_amount': trade.stake_amount, } if 'fiat_display_currency' in self.config: @@ -1650,41 +1718,52 @@ class FreqtradeBot(LoggingMixin): self.handle_order_fee(trade, order_obj, order) trade.update_trade(order_obj) - # TODO: is the below necessary? it's already done in update_trade for filled buys - trade.recalc_trade_from_orders() - Trade.commit() - if order['status'] in constants.NON_OPEN_EXCHANGE_STATES: + if order.get('status') in constants.NON_OPEN_EXCHANGE_STATES: # If a entry order was closed, force update on stoploss on exchange if order.get('side') == trade.entry_side: trade = self.cancel_stoploss_on_exchange(trade) - # TODO: Margin will need to use interest_rate as well. - # interest_rate = self.exchange.get_interest_rate() - trade.set_isolated_liq(self.exchange.get_liquidation_price( - leverage=trade.leverage, - pair=trade.pair, - amount=trade.amount, - open_rate=trade.open_rate, - is_short=trade.is_short - )) if not self.edge: # TODO: should shorting/leverage be supported by Edge, # then this will need to be fixed. trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) + if order.get('side') == trade.entry_side or trade.amount > 0: + # Must also run for partial exits + # TODO: Margin will need to use interest_rate as well. + # interest_rate = self.exchange.get_interest_rate() + trade.set_liquidation_price(self.exchange.get_liquidation_price( + leverage=trade.leverage, + pair=trade.pair, + amount=trade.amount, + stake_amount=trade.stake_amount, + open_rate=trade.open_rate, + is_short=trade.is_short + )) # Updating wallets when order is closed self.wallets.update() + Trade.commit() - if not trade.is_open: - if send_msg and not stoploss_order and not trade.open_order_id: - self._notify_exit(trade, '', True) - self.handle_protections(trade.pair, trade.trade_direction) - elif send_msg and not trade.open_order_id and not stoploss_order: - # Enter fill - self._notify_enter(trade, order, fill=True) + self.order_close_notify(trade, order_obj, stoploss_order, send_msg) return False + def order_close_notify( + self, trade: Trade, order: Order, stoploss_order: bool, send_msg: bool): + """send "fill" notifications""" + + sub_trade = not isclose(order.safe_amount_after_fee, + trade.amount, abs_tol=constants.MATH_CLOSE_PREC) + if order.ft_order_side == trade.exit_side: + # Exit notification + if send_msg and not stoploss_order and not trade.open_order_id: + self._notify_exit(trade, '', fill=True, sub_trade=sub_trade, order=order) + if not trade.is_open: + self.handle_protections(trade.pair, trade.trade_direction) + elif send_msg and not trade.open_order_id and not stoploss_order: + # Enter fill + self._notify_enter(trade, order, fill=True, sub_trade=sub_trade) + def handle_protections(self, pair: str, side: LongShort) -> None: prot_trig = self.protections.stop_per_pair(pair, side=side) if prot_trig: @@ -1806,6 +1885,9 @@ class FreqtradeBot(LoggingMixin): if fee_rate is not None and fee_rate < 0.02: # Only update if fee-rate is < 2% trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) + else: + logger.warning( + f"Not updating {order.get('side', '')}-fee - rate: {fee_rate}, {fee_currency}.") if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): # * Leverage could be a cause for this warning diff --git a/freqtrade/leverage/interest.py b/freqtrade/leverage/interest.py index 367df5821..ddeea2b42 100644 --- a/freqtrade/leverage/interest.py +++ b/freqtrade/leverage/interest.py @@ -1,20 +1,20 @@ -from decimal import Decimal from math import ceil from freqtrade.exceptions import OperationalException +from freqtrade.util import FtPrecise -one = Decimal(1.0) -four = Decimal(4.0) -twenty_four = Decimal(24.0) +one = FtPrecise(1.0) +four = FtPrecise(4.0) +twenty_four = FtPrecise(24.0) def interest( exchange_name: str, - borrowed: Decimal, - rate: Decimal, - hours: Decimal -) -> Decimal: + borrowed: FtPrecise, + rate: FtPrecise, + hours: FtPrecise +) -> FtPrecise: """ Equation to calculate interest on margin trades @@ -31,13 +31,13 @@ def interest( """ exchange_name = exchange_name.lower() if exchange_name == "binance": - return borrowed * rate * ceil(hours) / twenty_four + return borrowed * rate * FtPrecise(ceil(hours)) / twenty_four elif exchange_name == "kraken": # Rounded based on https://kraken-fees-calculator.github.io/ - return borrowed * rate * (one + ceil(hours / four)) + return borrowed * rate * (one + FtPrecise(ceil(hours / four))) elif exchange_name == "ftx": # As Explained under #Interest rates section in # https://help.ftx.com/hc/en-us/articles/360053007671-Spot-Margin-Trading-Explainer - return borrowed * rate * ceil(hours) / twenty_four + return borrowed * rate * FtPrecise(ceil(hours)) / twenty_four else: raise OperationalException(f"Leverage not available on {exchange_name} with freqtrade") diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py old mode 100755 new mode 100644 index 4d16dc0f1..57b272e86 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -23,7 +23,8 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType, RunMode, TradingMode) from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds +from freqtrade.exchange import (amount_to_contract_precision, price_to_precision, + timeframe_to_minutes, timeframe_to_seconds) from freqtrade.mixins import LoggingMixin from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.bt_progress import BTProgress @@ -89,6 +90,9 @@ class Backtesting: self.dataprovider = DataProvider(self.config, self.exchange) if self.config.get('strategy_list'): + if self.config.get('freqai', {}).get('enabled', False): + raise OperationalException( + "You can't use strategy_list and freqai at the same time.") for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat @@ -128,6 +132,7 @@ class Backtesting: self.fee = config['fee'] else: self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) + self.precision_mode = self.exchange.precisionMode self.timerange = TimeRange.parse_timerange( None if self.config.get('timerange') is None else str(self.config.get('timerange'))) @@ -207,6 +212,15 @@ class Backtesting: """ self.progress.init_step(BacktestState.DATALOAD, 1) + if self.config.get('freqai', {}).get('enabled', False): + startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0)) + if not startup_candles: + raise OperationalException('FreqAI backtesting module requires user set ' + 'startup_candles in config.') + self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0)) + logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}') + self.config['startup_candle_count'] = self.required_startup + data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, @@ -253,7 +267,7 @@ class Backtesting: funding_rates_dict = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, - timeframe=self.exchange._ft_has['mark_ohlcv_timeframe'], + timeframe=self.exchange.get_option('mark_ohlcv_timeframe'), timerange=self.timerange, startup_candles=0, fail_without_data=True, @@ -265,12 +279,12 @@ class Backtesting: mark_rates_dict = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, - timeframe=self.exchange._ft_has['mark_ohlcv_timeframe'], + timeframe=self.exchange.get_option('mark_ohlcv_timeframe'), timerange=self.timerange, startup_candles=0, fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), - candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"]) + candle_type=CandleType.from_string(self.exchange.get_option("mark_ohlcv_price")) ) # Combine data to avoid combining the data per trade. unavailable_pairs = [] @@ -287,8 +301,8 @@ class Backtesting: if unavailable_pairs: raise OperationalException( - f"Pairs {', '.join(unavailable_pairs)} got no leverage tiers available. " - "It is therefore impossible to backtest with this pair at the moment.") + f"Pairs {', '.join(unavailable_pairs)} got no leverage tiers available. " + "It is therefore impossible to backtest with this pair at the moment.") else: self.futures_data = {} @@ -381,7 +395,8 @@ class Backtesting: Get close rate for backtesting result """ # Special handling if high or low hit STOP_LOSS or ROI - if exit.exit_type in (ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS): + if exit.exit_type in ( + ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS, ExitType.LIQUIDATION): return self._get_close_rate_for_stoploss(row, trade, exit, trade_dur) elif exit.exit_type == (ExitType.ROI): return self._get_close_rate_for_roi(row, trade, exit, trade_dur) @@ -396,11 +411,16 @@ class Backtesting: is_short = trade.is_short or False leverage = trade.leverage or 1.0 side_1 = -1 if is_short else 1 + if exit.exit_type == ExitType.LIQUIDATION and trade.liquidation_price: + stoploss_value = trade.liquidation_price + else: + stoploss_value = trade.stop_loss + if is_short: - if trade.stop_loss < row[LOW_IDX]: + if stoploss_value < row[LOW_IDX]: return row[OPEN_IDX] else: - if trade.stop_loss > row[HIGH_IDX]: + if stoploss_value > row[HIGH_IDX]: return row[OPEN_IDX] # Special case: trailing triggers within same candle as trade opened. Assume most @@ -433,7 +453,7 @@ class Backtesting: return max(row[LOW_IDX], stop_rate) # Set close_rate to stoploss - return trade.stop_loss + return stoploss_value def _get_close_rate_for_roi(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, trade_dur: int) -> float: @@ -497,23 +517,50 @@ class Backtesting: def _get_adjust_trade_entry_for_candle(self, trade: LocalTrade, row: Tuple ) -> LocalTrade: - current_profit = trade.calc_profit_ratio(row[OPEN_IDX]) - min_stake = self.exchange.get_min_pair_stake_amount(trade.pair, row[OPEN_IDX], -0.1) - max_stake = self.exchange.get_max_pair_stake_amount(trade.pair, row[OPEN_IDX]) + current_rate = row[OPEN_IDX] + current_date = row[DATE_IDX].to_pydatetime() + current_profit = trade.calc_profit_ratio(current_rate) + min_stake = self.exchange.get_min_pair_stake_amount(trade.pair, current_rate, -0.1) + max_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_rate) stake_available = self.wallets.get_available_stake_amount() stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position, default_retval=None)( trade=trade, # type: ignore[arg-type] - current_time=row[DATE_IDX].to_pydatetime(), current_rate=row[OPEN_IDX], + current_time=current_date, current_rate=current_rate, current_profit=current_profit, min_stake=min_stake, - max_stake=min(max_stake, stake_available)) + max_stake=min(max_stake, stake_available), + current_entry_rate=current_rate, current_exit_rate=current_rate, + current_entry_profit=current_profit, current_exit_profit=current_profit) # Check if we should increase our position if stake_amount is not None and stake_amount > 0.0: + check_adjust_entry = True + if self.strategy.max_entry_position_adjustment > -1: + entry_count = trade.nr_of_successful_entries + check_adjust_entry = (entry_count <= self.strategy.max_entry_position_adjustment) + if check_adjust_entry: + pos_trade = self._enter_trade( + trade.pair, row, 'short' if trade.is_short else 'long', stake_amount, trade) + if pos_trade is not None: + self.wallets.update() + return pos_trade - pos_trade = self._enter_trade( - trade.pair, row, 'short' if trade.is_short else 'long', stake_amount, trade) + if stake_amount is not None and stake_amount < 0.0: + amount = abs(stake_amount) / current_rate + if amount > trade.amount: + # This is currently ineffective as remaining would become < min tradable + amount = trade.amount + remaining = (trade.amount - amount) * current_rate + if remaining < min_stake: + # Remaining stake is too low to be sold. + return trade + exit_ = ExitCheckTuple(ExitType.PARTIAL_EXIT) + pos_trade = self._get_exit_for_signal(trade, row, exit_, amount) if pos_trade is not None: + order = pos_trade.orders[-1] + if self._get_order_filled(order.price, row): + order.close_bt_order(current_date, trade) + trade.recalc_trade_from_orders() self.wallets.update() return pos_trade @@ -528,12 +575,7 @@ class Backtesting: # Check if we need to adjust our current positions if self.strategy.position_adjustment_enable: - check_adjust_entry = True - if self.strategy.max_entry_position_adjustment > -1: - entry_count = trade.nr_of_successful_entries - check_adjust_entry = (entry_count <= self.strategy.max_entry_position_adjustment) - if check_adjust_entry: - trade = self._get_adjust_trade_entry_for_candle(trade, row) + trade = self._get_adjust_trade_entry_for_candle(trade, row) enter = row[SHORT_IDX] if trade.is_short else row[LONG_IDX] exit_sig = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX] @@ -548,14 +590,15 @@ class Backtesting: return t return None - def _get_exit_for_signal(self, trade: LocalTrade, row: Tuple, - exit_: ExitCheckTuple) -> Optional[LocalTrade]: + def _get_exit_for_signal( + self, trade: LocalTrade, row: Tuple, exit_: ExitCheckTuple, + amount: Optional[float] = None) -> Optional[LocalTrade]: exit_candle_time: datetime = row[DATE_IDX].to_pydatetime() if exit_.exit_flag: trade.close_date = exit_candle_time exit_reason = exit_.exit_reason - + amount_ = amount if amount is not None else trade.amount trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) try: close_rate = self._get_close_rate(row, trade, exit_, trade_dur) @@ -564,10 +607,11 @@ class Backtesting: # call the custom exit price,with default value as previous close_rate current_profit = trade.calc_profit_ratio(close_rate) order_type = self.strategy.order_types['exit'] - if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT): + if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT, + ExitType.PARTIAL_EXIT): # Checks and adds an exit tag, after checking that the length of the # row has the length for an exit tag column - if( + if ( len(row) > EXIT_TAG_IDX and row[EXIT_TAG_IDX] is not None and len(row[EXIT_TAG_IDX]) > 0 @@ -592,46 +636,57 @@ class Backtesting: # Confirm trade exit: time_in_force = self.strategy.order_time_in_force['exit'] - if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( - pair=trade.pair, - trade=trade, # type: ignore[arg-type] - order_type='limit', - amount=trade.amount, - rate=close_rate, - time_in_force=time_in_force, - sell_reason=exit_reason, # deprecated - exit_reason=exit_reason, - current_time=exit_candle_time): + if (exit_.exit_type not in (ExitType.LIQUIDATION, ExitType.PARTIAL_EXIT) + and not strategy_safe_wrapper( + self.strategy.confirm_trade_exit, default_retval=True)( + pair=trade.pair, + trade=trade, # type: ignore[arg-type] + order_type=order_type, + amount=amount_, + rate=close_rate, + time_in_force=time_in_force, + sell_reason=exit_reason, # deprecated + exit_reason=exit_reason, + current_time=exit_candle_time)): return None trade.exit_reason = exit_reason - self.order_id_counter += 1 - order = Order( - id=self.order_id_counter, - ft_trade_id=trade.id, - order_date=exit_candle_time, - order_update_date=exit_candle_time, - ft_is_open=True, - ft_pair=trade.pair, - order_id=str(self.order_id_counter), - symbol=trade.pair, - ft_order_side=trade.exit_side, - side=trade.exit_side, - order_type=order_type, - status="open", - price=close_rate, - average=close_rate, - amount=trade.amount, - filled=0, - remaining=trade.amount, - cost=trade.amount * close_rate, - ) - trade.orders.append(order) - return trade - + return self._exit_trade(trade, row, close_rate, amount_) return None + def _exit_trade(self, trade: LocalTrade, sell_row: Tuple, + close_rate: float, amount: float = None) -> Optional[LocalTrade]: + self.order_id_counter += 1 + exit_candle_time = sell_row[DATE_IDX].to_pydatetime() + order_type = self.strategy.order_types['exit'] + # amount = amount or trade.amount + amount = amount_to_contract_precision(amount or trade.amount, trade.amount_precision, + self.precision_mode, trade.contract_size) + rate = price_to_precision(close_rate, trade.price_precision, self.precision_mode) + order = Order( + id=self.order_id_counter, + ft_trade_id=trade.id, + order_date=exit_candle_time, + order_update_date=exit_candle_time, + ft_is_open=True, + ft_pair=trade.pair, + order_id=str(self.order_id_counter), + symbol=trade.pair, + ft_order_side=trade.exit_side, + side=trade.exit_side, + order_type=order_type, + status="open", + price=rate, + average=rate, + amount=amount, + filled=0, + remaining=amount, + cost=amount * rate, + ) + trade.orders.append(order) + return trade + def _get_exit_trade_entry(self, trade: LocalTrade, row: Tuple) -> Optional[LocalTrade]: exit_candle_time: datetime = row[DATE_IDX].to_pydatetime() @@ -773,7 +828,17 @@ class Backtesting: if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount): self.order_id_counter += 1 base_currency = self.exchange.get_pair_base_currency(pair) - amount = round((stake_amount / propose_rate) * leverage, 8) + precision_price = self.exchange.get_precision_price(pair) + propose_rate = price_to_precision(propose_rate, precision_price, self.precision_mode) + amount_p = (stake_amount / propose_rate) * leverage + + contract_size = self.exchange.get_contract_size(pair) + precision_amount = self.exchange.get_precision_amount(pair) + amount = amount_to_contract_precision(amount_p, precision_amount, self.precision_mode, + contract_size) + # Backcalculate actual stake amount. + stake_amount = amount * propose_rate / leverage + is_short = (direction == 'short') # Necessary for Margin trading. Disabled until support is enabled. # interest_rate = self.exchange.get_interest_rate() @@ -802,15 +867,20 @@ class Backtesting: trading_mode=self.trading_mode, leverage=leverage, # interest_rate=interest_rate, + amount_precision=precision_amount, + price_precision=precision_price, + precision_mode=self.precision_mode, + contract_size=contract_size, orders=[], ) trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) - trade.set_isolated_liq(self.exchange.get_liquidation_price( + trade.set_liquidation_price(self.exchange.get_liquidation_price( pair=pair, open_rate=propose_rate, amount=amount, + stake_amount=trade.stake_amount, leverage=leverage, is_short=is_short, )) @@ -858,6 +928,8 @@ class Backtesting: # Ignore trade if entry-order did not fill yet continue exit_row = data[pair][-1] + self._exit_trade(trade, exit_row, exit_row[OPEN_IDX], trade.amount) + trade.orders[-1].close_bt_order(exit_row[DATE_IDX].to_pydatetime(), trade) trade.close_date = exit_row[DATE_IDX].to_pydatetime() trade.exit_reason = ExitType.FORCE_EXIT.value @@ -999,7 +1071,7 @@ class Backtesting: return None return row - def backtest(self, processed: Dict, + def backtest(self, processed: Dict, # noqa: max-complexity: 13 start_date: datetime, end_date: datetime, max_open_trades: int = 0, position_stacking: bool = False, enable_protections: bool = False) -> Dict[str, Any]: @@ -1101,14 +1173,19 @@ class Backtesting: if order and self._get_order_filled(order.price, row): order.close_bt_order(current_time, trade) trade.open_order_id = None - trade.close_date = current_time - trade.close(order.price, show_msg=False) + sub_trade = order.safe_amount_after_fee != trade.amount + if sub_trade: + order.close_bt_order(current_time, trade) + trade.recalc_trade_from_orders() + else: + trade.close_date = current_time + trade.close(order.price, show_msg=False) - # logger.debug(f"{pair} - Backtesting exit {trade}") - open_trade_count -= 1 - open_trades[pair].remove(trade) - LocalTrade.close_bt_trade(trade) - trades.append(trade) + # logger.debug(f"{pair} - Backtesting exit {trade}") + open_trade_count -= 1 + open_trades[pair].remove(trade) + LocalTrade.close_bt_trade(trade) + trades.append(trade) self.wallets.update() self.run_protections( enable_protections, pair, current_time, trade.trade_direction) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 566412f29..fea2a672f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -24,13 +24,15 @@ from pandas import DataFrame from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange +from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, file_dump_json, plural from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss -from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer +from freqtrade.optimize.hyperopt_tools import (HyperoptStateContainer, HyperoptTools, + hyperopt_serializer) from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver @@ -74,10 +76,14 @@ class Hyperopt: self.dimensions: List[Dimension] = [] self.config = config + self.min_date: datetime + self.max_date: datetime self.backtesting = Backtesting(self.config) self.pairlist = self.backtesting.pairlists.whitelist self.custom_hyperopt: HyperOptAuto + self.analyze_per_epoch = self.config.get('analyze_per_epoch', False) + HyperoptStateContainer.set_state(HyperoptState.STARTUP) if not self.config.get('hyperopt'): self.custom_hyperopt = HyperOptAuto(self.config) @@ -290,6 +296,7 @@ class Hyperopt: Called once per epoch to optimize whatever is configured. Keep this function as optimized as possible! """ + HyperoptStateContainer.set_state(HyperoptState.OPTIMIZE) backtest_start_time = datetime.now(timezone.utc) params_dict = self._get_params_dict(self.dimensions, raw_params) @@ -321,6 +328,10 @@ class Hyperopt: with self.data_pickle_file.open('rb') as f: processed = load(f, mmap_mode='r') + if self.analyze_per_epoch: + # Data is not yet analyzed, rerun populate_indicators. + processed = self.advise_and_trim(processed) + bt_results = self.backtesting.backtest( processed=processed, start_date=self.min_date, @@ -406,22 +417,33 @@ class Hyperopt: def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) - def prepare_hyperopt_data(self) -> None: - data, timerange = self.backtesting.load_bt_data() - self.backtesting.load_bt_data_detail() - logger.info("Dataload complete. Calculating indicators") - + def advise_and_trim(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]: preprocessed = self.backtesting.strategy.advise_all_indicators(data) # Trim startup period from analyzed dataframe to get correct dates for output. - processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup) + processed = trim_dataframes(preprocessed, self.timerange, self.backtesting.required_startup) self.min_date, self.max_date = get_timerange(processed) + return processed - logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'({(self.max_date - self.min_date).days} days)..') - # Store non-trimmed data - will be trimmed after signal generation. - dump(preprocessed, self.data_pickle_file) + def prepare_hyperopt_data(self) -> None: + HyperoptStateContainer.set_state(HyperoptState.DATALOAD) + data, self.timerange = self.backtesting.load_bt_data() + self.backtesting.load_bt_data_detail() + logger.info("Dataload complete. Calculating indicators") + + if not self.analyze_per_epoch: + HyperoptStateContainer.set_state(HyperoptState.INDICATORS) + + preprocessed = self.advise_and_trim(data) + + logger.info(f'Hyperopting with data from ' + f'{self.min_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'({(self.max_date - self.min_date).days} days)..') + # Store non-trimmed data - will be trimmed after signal generation. + dump(preprocessed, self.data_pickle_file) + else: + dump(data, self.data_pickle_file) def get_asked_points(self, n_points: int) -> Tuple[List[List[Any]], List[bool]]: """ @@ -483,6 +505,7 @@ class Hyperopt: self.backtesting.exchange._api_async = None self.backtesting.exchange.loop = None # type: ignore self.backtesting.exchange._loop_lock = None # type: ignore + self.backtesting.exchange._cache_lock = None # type: ignore # self.backtesting.exchange = None # type: ignore self.backtesting.pairlists = None # type: ignore diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index ab6ef013b..9b022d519 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -13,6 +13,7 @@ from colorama import Fore, Style from pandas import isna, json_normalize from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES +from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2 from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs @@ -32,6 +33,15 @@ def hyperopt_serializer(x): return str(x) +class HyperoptStateContainer(): + """ Singleton class to track state of hyperopt""" + state: HyperoptState = HyperoptState.OPTIMIZE + + @classmethod + def set_state(cls, value: HyperoptState): + cls.state = value + + class HyperoptTools(): @staticmethod diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 44ac4a5b3..519022db2 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -639,7 +639,7 @@ def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_curr :param stake_currency: stake-currency - used to correctly name headers :return: pretty printed table with tabulate as string """ - if(tag_type == "enter_tag"): + if (tag_type == "enter_tag"): headers = _get_line_header("TAG", stake_currency) else: headers = _get_line_header("TAG", stake_currency, 'Sells') diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index f4e7470a7..9e1a7e922 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -1,5 +1,5 @@ # flake8: noqa: F401 -from freqtrade.persistence.models import cleanup_db, init_db +from freqtrade.persistence.models import init_db from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import LocalTrade, Order, Trade diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 2a8e34cdf..1131c88b4 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -95,6 +95,7 @@ def migrate_trades_and_orders_table( exit_reason = get_column_def(cols, 'sell_reason', get_column_def(cols, 'exit_reason', 'null')) strategy = get_column_def(cols, 'strategy', 'null') enter_tag = get_column_def(cols, 'buy_tag', get_column_def(cols, 'enter_tag', 'null')) + realized_profit = get_column_def(cols, 'realized_profit', '0.0') trading_mode = get_column_def(cols, 'trading_mode', 'null') @@ -129,6 +130,11 @@ def migrate_trades_and_orders_table( get_column_def(cols, 'sell_order_status', 'null')) amount_requested = get_column_def(cols, 'amount_requested', 'amount') + amount_precision = get_column_def(cols, 'amount_precision', 'null') + price_precision = get_column_def(cols, 'price_precision', 'null') + precision_mode = get_column_def(cols, 'precision_mode', 'null') + contract_size = get_column_def(cols, 'contract_size', 'null') + # Schema migration necessary with engine.begin() as connection: connection.execute(text(f"alter table trades rename to {trade_back_name}")) @@ -155,7 +161,8 @@ def migrate_trades_and_orders_table( max_rate, min_rate, exit_reason, exit_order_status, strategy, enter_tag, timeframe, open_trade_value, close_profit_abs, trading_mode, leverage, liquidation_price, is_short, - interest_rate, funding_fees + interest_rate, funding_fees, realized_profit, + amount_precision, price_precision, precision_mode, contract_size ) select id, lower(exchange), pair, {base_currency} base_currency, {stake_currency} stake_currency, @@ -181,7 +188,9 @@ def migrate_trades_and_orders_table( {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs, {trading_mode} trading_mode, {leverage} leverage, {liquidation_price} liquidation_price, {is_short} is_short, {interest_rate} interest_rate, - {funding_fees} funding_fees + {funding_fees} funding_fees, {realized_profit} realized_profit, + {amount_precision} amount_precision, {price_precision} price_precision, + {precision_mode} precision_mode, {contract_size} contract_size from {trade_back_name} """)) @@ -297,8 +306,11 @@ def check_migrate(engine, decl_base, previous_tables) -> None: # Check if migration necessary # Migrates both trades and orders table! - if not has_column(cols_orders, 'stop_price'): - # if not has_column(cols_trades, 'base_currency'): + # if ('orders' not in previous_tables + # or not has_column(cols_orders, 'stop_price')): + migrating = False + if not has_column(cols_trades, 'contract_size'): + migrating = True logger.info(f"Running database migration for trades - " f"backup: {table_back_name}, {order_table_bak_name}") migrate_trades_and_orders_table( @@ -306,6 +318,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None: order_table_bak_name, cols_orders) if not has_column(cols_pairlocks, 'side'): + migrating = True logger.info(f"Running database migration for pairlocks - " f"backup: {pairlock_table_bak_name}") @@ -320,3 +333,6 @@ def check_migrate(engine, decl_base, previous_tables) -> None: set_sqlite_to_wal(engine) fix_old_dry_orders(engine) + + if migrating: + logger.info("Database migration finished.") diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 86d2f9f9c..7f851322e 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -53,7 +53,7 @@ def init_db(db_url: str) -> None: # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope # Scoped sessions proxy requests to the appropriate thread-local session. # We should use the scoped_session object - not a seperately initialized version - Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True)) + Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=False)) Trade.query = Trade._session.query_property() Order.query = Trade._session.query_property() PairLock.query = Trade._session.query_property() @@ -61,11 +61,3 @@ def init_db(db_url: str) -> None: previous_tables = inspect(engine).get_table_names() _DECL_BASE.metadata.create_all(engine) check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables) - - -def cleanup_db() -> None: - """ - Flushes all pending operations to disk. - :return: None - """ - Trade.commit() diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 5f302de71..23997f835 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -3,18 +3,21 @@ This module contains the class to persist trades into SQLite """ import logging from datetime import datetime, timedelta, timezone -from decimal import Decimal +from math import isclose from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func) from sqlalchemy.orm import Query, lazyload, relationship -from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort +from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES, + BuySell, LongShort) from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException +from freqtrade.exchange import amount_to_contract_precision, price_to_precision from freqtrade.leverage import interest from freqtrade.persistence.base import _DECL_BASE +from freqtrade.util import FtPrecise logger = logging.getLogger(__name__) @@ -176,10 +179,9 @@ class Order(_DECL_BASE): self.remaining = 0 self.status = 'closed' self.ft_is_open = False - if (self.ft_order_side == trade.entry_side - and len(trade.select_filled_orders(trade.entry_side)) == 1): + if (self.ft_order_side == trade.entry_side): trade.open_rate = self.price - trade.recalc_open_trade_value() + trade.recalc_trade_from_orders() trade.adjust_stop_loss(trade.open_rate, trade.stop_loss_pct, refresh=True) @staticmethod @@ -195,7 +197,7 @@ class Order(_DECL_BASE): if filtered_orders: oobj = filtered_orders[0] oobj.update_from_ccxt_object(order) - Order.query.session.commit() + Trade.commit() else: logger.warning(f"Did not find order for {order}.") @@ -237,6 +239,7 @@ class LocalTrade(): trades: List['LocalTrade'] = [] trades_open: List['LocalTrade'] = [] total_profit: float = 0 + realized_profit: float = 0 id: int = 0 @@ -290,6 +293,10 @@ class LocalTrade(): timeframe: Optional[int] = None trading_mode: TradingMode = TradingMode.SPOT + amount_precision: Optional[float] = None + price_precision: Optional[float] = None + precision_mode: Optional[int] = None + contract_size: Optional[float] = None # Leverage trading properties liquidation_price: Optional[float] = None @@ -302,6 +309,16 @@ class LocalTrade(): # Futures properties funding_fees: Optional[float] = None + @property + def stoploss_or_liquidation(self) -> float: + if self.liquidation_price: + if self.is_short: + return min(self.stop_loss, self.liquidation_price) + else: + return max(self.stop_loss, self.liquidation_price) + + return self.stop_loss + @property def buy_tag(self) -> Optional[str]: """ @@ -437,6 +454,7 @@ class LocalTrade(): if self.close_date else None), 'close_timestamp': int(self.close_date.replace( tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None, + 'realized_profit': self.realized_profit or 0.0, 'close_rate': self.close_rate, 'close_rate_requested': self.close_rate_requested, 'close_profit': self.close_profit, # Deprecated @@ -497,7 +515,7 @@ class LocalTrade(): self.max_rate = max(current_price, self.max_rate or self.open_rate) self.min_rate = min(current_price_low, self.min_rate or self.open_rate) - def set_isolated_liq(self, liquidation_price: Optional[float]): + def set_liquidation_price(self, liquidation_price: Optional[float]): """ Method you should use to set self.liquidation price. Assures stop_loss is not passed the liquidation price @@ -506,22 +524,14 @@ class LocalTrade(): return self.liquidation_price = liquidation_price - def _set_stop_loss(self, stop_loss: float, percent: float): + def __set_stop_loss(self, stop_loss: float, percent: float): """ - Method you should use to set self.stop_loss. - Assures stop_loss is not passed the liquidation price + Method used internally to set self.stop_loss. """ - if self.liquidation_price is not None: - if self.is_short: - sl = min(stop_loss, self.liquidation_price) - else: - sl = max(stop_loss, self.liquidation_price) - else: - sl = stop_loss - + stop_loss_norm = price_to_precision(stop_loss, self.price_precision, self.precision_mode) if not self.stop_loss: - self.initial_stop_loss = sl - self.stop_loss = sl + self.initial_stop_loss = stop_loss_norm + self.stop_loss = stop_loss_norm self.stop_loss_pct = -1 * abs(percent) self.stoploss_last_update = datetime.utcnow() @@ -543,19 +553,14 @@ class LocalTrade(): leverage = self.leverage or 1.0 if self.is_short: new_loss = float(current_price * (1 + abs(stoploss / leverage))) - # If trading with leverage, don't set the stoploss below the liquidation price - if self.liquidation_price: - new_loss = min(self.liquidation_price, new_loss) else: new_loss = float(current_price * (1 - abs(stoploss / leverage))) - # If trading with leverage, don't set the stoploss below the liquidation price - if self.liquidation_price: - new_loss = max(self.liquidation_price, new_loss) # no stop loss assigned yet if self.initial_stop_loss_pct is None or refresh: - self._set_stop_loss(new_loss, stoploss) - self.initial_stop_loss = new_loss + self.__set_stop_loss(new_loss, stoploss) + self.initial_stop_loss = price_to_precision( + new_loss, self.price_precision, self.precision_mode) self.initial_stop_loss_pct = -1 * abs(stoploss) # evaluate if the stop loss needs to be updated @@ -569,7 +574,7 @@ class LocalTrade(): # ? decreasing the minimum stoploss if (higher_stop and not self.is_short) or (lower_stop and self.is_short): logger.debug(f"{self.pair} - Adjusting stoploss...") - self._set_stop_loss(new_loss, stoploss) + self.__set_stop_loss(new_loss, stoploss) else: logger.debug(f"{self.pair} - Keeping current stoploss...") @@ -601,14 +606,30 @@ class LocalTrade(): if self.is_open: payment = "SELL" if self.is_short else "BUY" logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.') - self.open_order_id = None + # condition to avoid reset value when updating fees + if self.open_order_id == order.order_id: + self.open_order_id = None + else: + logger.warning( + f'Got different open_order_id {self.open_order_id} != {order.order_id}') self.recalc_trade_from_orders() elif order.ft_order_side == self.exit_side: if self.is_open: payment = "BUY" if self.is_short else "SELL" # * On margin shorts, you buy a little bit more than the amount (amount + interest) logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.') - self.close(order.safe_price) + # condition to avoid reset value when updating fees + if self.open_order_id == order.order_id: + self.open_order_id = None + else: + logger.warning( + f'Got different open_order_id {self.open_order_id} != {order.order_id}') + amount_tr = amount_to_contract_precision(self.amount, self.amount_precision, + self.precision_mode, self.contract_size) + if isclose(order.safe_amount_after_fee, amount_tr, abs_tol=MATH_CLOSE_PREC): + self.close(order.safe_price) + else: + self.recalc_trade_from_orders() elif order.ft_order_side == 'stoploss': self.stoploss_order_id = None self.close_rate_requested = self.stop_loss @@ -627,11 +648,11 @@ class LocalTrade(): """ self.close_rate = rate self.close_date = self.close_date or datetime.utcnow() - self.close_profit = self.calc_profit_ratio(rate) - self.close_profit_abs = self.calc_profit(rate) + self.close_profit_abs = self.calc_profit(rate) + self.realized_profit self.is_open = False self.exit_order_status = 'closed' self.open_order_id = None + self.recalc_trade_from_orders(is_closing=True) if show_msg: logger.info( 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', @@ -677,13 +698,13 @@ class LocalTrade(): """ return len([o for o in self.orders if o.ft_order_side == self.exit_side]) - def _calc_open_trade_value(self) -> float: + def _calc_open_trade_value(self, amount: float, open_rate: float) -> float: """ Calculate the open_rate including open_fee. :return: Price in of the open trade incl. Fees """ - open_trade = Decimal(self.amount) * Decimal(self.open_rate) - fees = open_trade * Decimal(self.fee_open) + open_trade = FtPrecise(amount) * FtPrecise(open_rate) + fees = open_trade * FtPrecise(self.fee_open) if self.is_short: return float(open_trade - fees) else: @@ -694,39 +715,39 @@ class LocalTrade(): Recalculate open_trade_value. Must be called whenever open_rate, fee_open is changed. """ - self.open_trade_value = self._calc_open_trade_value() + self.open_trade_value = self._calc_open_trade_value(self.amount, self.open_rate) - def calculate_interest(self) -> Decimal: + def calculate_interest(self) -> FtPrecise: """ Calculate interest for this trade. Only applicable for Margin trading. """ - zero = Decimal(0.0) + zero = FtPrecise(0.0) # If nothing was borrowed if self.trading_mode != TradingMode.MARGIN or self.has_no_leverage: return zero open_date = self.open_date.replace(tzinfo=None) now = (self.close_date or datetime.now(timezone.utc)).replace(tzinfo=None) - sec_per_hour = Decimal(3600) - total_seconds = Decimal((now - open_date).total_seconds()) + sec_per_hour = FtPrecise(3600) + total_seconds = FtPrecise((now - open_date).total_seconds()) hours = total_seconds / sec_per_hour or zero - rate = Decimal(self.interest_rate) - borrowed = Decimal(self.borrowed) + rate = FtPrecise(self.interest_rate) + borrowed = FtPrecise(self.borrowed) return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours) - def _calc_base_close(self, amount: Decimal, rate: float, fee: float) -> Decimal: + def _calc_base_close(self, amount: FtPrecise, rate: float, fee: float) -> FtPrecise: - close_trade = amount * Decimal(rate) - fees = close_trade * Decimal(fee) + close_trade = amount * FtPrecise(rate) + fees = close_trade * FtPrecise(fee) if self.is_short: return close_trade + fees else: return close_trade - fees - def calc_close_trade_value(self, rate: float) -> float: + def calc_close_trade_value(self, rate: float, amount: float = None) -> float: """ Calculate the Trade's close value including fees :param rate: rate to compare with. @@ -735,96 +756,145 @@ class LocalTrade(): if rate is None and not self.close_rate: return 0.0 - amount = Decimal(self.amount) + amount1 = FtPrecise(amount or self.amount) trading_mode = self.trading_mode or TradingMode.SPOT if trading_mode == TradingMode.SPOT: - return float(self._calc_base_close(amount, rate, self.fee_close)) + return float(self._calc_base_close(amount1, rate, self.fee_close)) elif (trading_mode == TradingMode.MARGIN): total_interest = self.calculate_interest() if self.is_short: - amount = amount + total_interest - return float(self._calc_base_close(amount, rate, self.fee_close)) + amount1 = amount1 + total_interest + return float(self._calc_base_close(amount1, rate, self.fee_close)) else: # Currency already owned for longs, no need to purchase - return float(self._calc_base_close(amount, rate, self.fee_close) - total_interest) + return float(self._calc_base_close(amount1, rate, self.fee_close) - total_interest) elif (trading_mode == TradingMode.FUTURES): funding_fees = self.funding_fees or 0.0 # Positive funding_fees -> Trade has gained from fees. # Negative funding_fees -> Trade had to pay the fees. if self.is_short: - return float(self._calc_base_close(amount, rate, self.fee_close)) - funding_fees + return float(self._calc_base_close(amount1, rate, self.fee_close)) - funding_fees else: - return float(self._calc_base_close(amount, rate, self.fee_close)) + funding_fees + return float(self._calc_base_close(amount1, rate, self.fee_close)) + funding_fees else: raise OperationalException( f"{self.trading_mode.value} trading is not yet available using freqtrade") - def calc_profit(self, rate: float) -> float: + def calc_profit(self, rate: float, amount: float = None, open_rate: float = None) -> float: """ Calculate the absolute profit in stake currency between Close and Open trade :param rate: close rate to compare with. + :param amount: Amount to use for the calculation. Falls back to trade.amount if not set. + :param open_rate: open_rate to use. Defaults to self.open_rate if not provided. :return: profit in stake currency as float """ - close_trade_value = self.calc_close_trade_value(rate) + close_trade_value = self.calc_close_trade_value(rate, amount) + if amount is None or open_rate is None: + open_trade_value = self.open_trade_value + else: + open_trade_value = self._calc_open_trade_value(amount, open_rate) if self.is_short: - profit = self.open_trade_value - close_trade_value + profit = open_trade_value - close_trade_value else: - profit = close_trade_value - self.open_trade_value + profit = close_trade_value - open_trade_value return float(f"{profit:.8f}") - def calc_profit_ratio(self, rate: float) -> float: + def calc_profit_ratio( + self, rate: float, amount: float = None, open_rate: float = None) -> float: """ Calculates the profit as ratio (including fee). :param rate: rate to compare with. + :param amount: Amount to use for the calculation. Falls back to trade.amount if not set. + :param open_rate: open_rate to use. Defaults to self.open_rate if not provided. :return: profit ratio as float """ - close_trade_value = self.calc_close_trade_value(rate) + close_trade_value = self.calc_close_trade_value(rate, amount) + + if amount is None or open_rate is None: + open_trade_value = self.open_trade_value + else: + open_trade_value = self._calc_open_trade_value(amount, open_rate) short_close_zero = (self.is_short and close_trade_value == 0.0) - long_close_zero = (not self.is_short and self.open_trade_value == 0.0) + long_close_zero = (not self.is_short and open_trade_value == 0.0) leverage = self.leverage or 1.0 if (short_close_zero or long_close_zero): return 0.0 else: if self.is_short: - profit_ratio = (1 - (close_trade_value / self.open_trade_value)) * leverage + profit_ratio = (1 - (close_trade_value / open_trade_value)) * leverage else: - profit_ratio = ((close_trade_value / self.open_trade_value) - 1) * leverage + profit_ratio = ((close_trade_value / open_trade_value) - 1) * leverage return float(f"{profit_ratio:.8f}") - def recalc_trade_from_orders(self): - - total_amount = 0.0 - total_stake = 0.0 + def recalc_trade_from_orders(self, *, is_closing: bool = False): + ZERO = FtPrecise(0.0) + current_amount = FtPrecise(0.0) + current_stake = FtPrecise(0.0) + total_stake = 0.0 # Total stake after all buy orders (does not subtract!) + avg_price = FtPrecise(0.0) + close_profit = 0.0 + close_profit_abs = 0.0 + profit = None for o in self.orders: - if (o.ft_is_open or - (o.ft_order_side != self.entry_side) or - (o.status not in NON_OPEN_EXCHANGE_STATES)): + if o.ft_is_open or not o.filled: continue - tmp_amount = o.safe_amount_after_fee - tmp_price = o.average or o.price - if tmp_amount > 0.0 and tmp_price is not None: - total_amount += tmp_amount - total_stake += tmp_price * tmp_amount + tmp_amount = FtPrecise(o.safe_amount_after_fee) + tmp_price = FtPrecise(o.safe_price) - if total_amount > 0: + is_exit = o.ft_order_side != self.entry_side + side = FtPrecise(-1 if is_exit else 1) + if tmp_amount > ZERO and tmp_price is not None: + current_amount += tmp_amount * side + price = avg_price if is_exit else tmp_price + current_stake += price * tmp_amount * side + + if current_amount > ZERO: + avg_price = current_stake / current_amount + + if is_exit: + # Process partial exits + exit_rate = o.safe_price + exit_amount = o.safe_amount_after_fee + profit = self.calc_profit(rate=exit_rate, amount=exit_amount, + open_rate=float(avg_price)) + close_profit_abs += profit + close_profit = self.calc_profit_ratio( + exit_rate, amount=exit_amount, open_rate=avg_price) + else: + total_stake = total_stake + self._calc_open_trade_value(tmp_amount, price) + + if close_profit: + self.close_profit = close_profit + self.realized_profit = close_profit_abs + self.close_profit_abs = profit + + current_amount_tr = amount_to_contract_precision( + float(current_amount), self.amount_precision, self.precision_mode, self.contract_size) + if current_amount_tr > 0.0: + # Trade is still open # Leverage not updated, as we don't allow changing leverage through DCA at the moment. - self.open_rate = total_stake / total_amount - self.stake_amount = total_stake / (self.leverage or 1.0) - self.amount = total_amount - self.fee_open_cost = self.fee_open * total_stake + self.open_rate = float(current_stake / current_amount) + self.amount = current_amount_tr + self.stake_amount = float(current_stake) / (self.leverage or 1.0) + self.fee_open_cost = self.fee_open * float(current_stake) self.recalc_open_trade_value() if self.stop_loss_pct is not None and self.open_rate is not None: self.adjust_stop_loss(self.open_rate, self.stop_loss_pct) + elif is_closing and total_stake > 0: + # Close profit abs / maximum owned + # Fees are considered as they are part of close_profit_abs + self.close_profit = (close_profit_abs / total_stake) * self.leverage + self.close_profit_abs = close_profit_abs def select_order_by_order_id(self, order_id: str) -> Optional[Order]: """ @@ -846,7 +916,7 @@ class LocalTrade(): """ orders = self.orders if order_side: - orders = [o for o in self.orders if o.ft_order_side == order_side] + orders = [o for o in orders if o.ft_order_side == order_side] if is_open is not None: orders = [o for o in orders if o.ft_is_open == is_open] if len(orders) > 0: @@ -861,9 +931,9 @@ class LocalTrade(): :return: array of Order objects """ return [o for o in self.orders if ((o.ft_order_side == order_side) or (order_side is None)) - and o.ft_is_open is False and - (o.filled or 0) > 0 and - o.status in NON_OPEN_EXCHANGE_STATES] + and o.ft_is_open is False + and o.filled + and o.status in NON_OPEN_EXCHANGE_STATES] def select_filled_or_open_orders(self) -> List['Order']: """ @@ -975,6 +1045,16 @@ class LocalTrade(): """ return Trade.get_trades_proxy(is_open=True) + @staticmethod + def get_open_trade_count() -> int: + """ + get open trade count + """ + if Trade.use_db: + return Trade.query.filter(Trade.is_open.is_(True)).count() + else: + return len(LocalTrade.trades_open) + @staticmethod def stoploss_reinitialization(desired_stoploss): """ @@ -1028,6 +1108,7 @@ class Trade(_DECL_BASE, LocalTrade): open_trade_value = Column(Float) close_rate: Optional[float] = Column(Float) close_rate_requested = Column(Float) + realized_profit = Column(Float, default=0.0) close_profit = Column(Float) close_profit_abs = Column(Float) stake_amount = Column(Float, nullable=False) @@ -1059,6 +1140,10 @@ class Trade(_DECL_BASE, LocalTrade): timeframe = Column(Integer, nullable=True) trading_mode = Column(Enum(TradingMode), nullable=True) + amount_precision = Column(Float, nullable=True) + price_precision = Column(Float, nullable=True) + precision_mode = Column(Integer, nullable=True) + contract_size = Column(Float, nullable=True) # Leverage trading properties leverage = Column(Float, nullable=True, default=1.0) @@ -1073,6 +1158,7 @@ class Trade(_DECL_BASE, LocalTrade): def __init__(self, **kwargs): super().__init__(**kwargs) + self.realized_profit = 0 self.recalc_open_trade_value() def delete(self) -> None: @@ -1087,6 +1173,10 @@ class Trade(_DECL_BASE, LocalTrade): def commit(): Trade.query.session.commit() + @staticmethod + def rollback(): + Trade.query.session.rollback() + @staticmethod def get_trades_proxy(*, pair: str = None, is_open: bool = None, open_date: datetime = None, close_date: datetime = None, @@ -1239,7 +1329,7 @@ class Trade(_DECL_BASE, LocalTrade): """ filters = [Trade.is_open.is_(False)] - if(pair is not None): + if (pair is not None): filters.append(Trade.pair == pair) enter_tag_perf = Trade.query.with_entities( @@ -1272,7 +1362,7 @@ class Trade(_DECL_BASE, LocalTrade): """ filters = [Trade.is_open.is_(False)] - if(pair is not None): + if (pair is not None): filters.append(Trade.pair == pair) sell_tag_perf = Trade.query.with_entities( @@ -1305,7 +1395,7 @@ class Trade(_DECL_BASE, LocalTrade): """ filters = [Trade.is_open.is_(False)] - if(pair is not None): + if (pair is not None): filters.append(Trade.pair == pair) mix_tag_perf = Trade.query.with_entities( @@ -1325,7 +1415,7 @@ class Trade(_DECL_BASE, LocalTrade): enter_tag = enter_tag if enter_tag is not None else "Other" exit_reason = exit_reason if exit_reason is not None else "Other" - if(exit_reason is not None and enter_tag is not None): + if (exit_reason is not None and enter_tag is not None): mix_tag = enter_tag + " " + exit_reason i = 0 if not any(item["mix_tag"] == mix_tag for item in return_list): diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 786f32e88..13c992c87 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -8,11 +8,11 @@ from typing import Any, Dict, List, Optional import arrow from pandas import DataFrame -from freqtrade.configuration import PeriodicCache from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.util import PeriodicCache logger = logging.getLogger(__name__) diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 521f38635..dcd153d8e 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -51,6 +51,11 @@ class PrecisionFilter(IPairList): :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ + if ticker.get('last', None) is None: + self.log_once(f"Removed {ticker['symbol']} from whitelist, because " + "ticker['last'] is empty (Usually no trade in the last 24h).", + logger.info) + return False stop_price = ticker['last'] * self._stoploss # Adjust stop-prices to precision diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index cd16a46a3..8138a5fb6 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -4,14 +4,14 @@ Volume PairList provider Provides dynamic pair list based on trade volumes """ import logging +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List -import arrow from cachetools import TTLCache from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException -from freqtrade.exchange import timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.misc import format_ms_time from freqtrade.plugins.pairlist.IPairList import IPairList @@ -73,7 +73,7 @@ class VolumePairList(IPairList): if (not self._use_range and not ( self._exchange.exchange_has('fetchTickers') - and self._exchange._ft_has["tickers_have_quoteVolume"])): + and self._exchange.get_option("tickers_have_quoteVolume"))): raise OperationalException( "Exchange does not support dynamic whitelist in this configuration. " "Please edit your config and either remove Volumepairlist, " @@ -158,16 +158,16 @@ class VolumePairList(IPairList): filtered_tickers: List[Dict[str, Any]] = [{'symbol': k} for k in pairlist] # get lookback period in ms, for exchange ohlcv fetch - since_ms = int(arrow.utcnow() - .floor('minute') - .shift(minutes=-(self._lookback_period * self._tf_in_min) - - self._tf_in_min) - .int_timestamp) * 1000 + since_ms = int(timeframe_to_prev_date( + self._lookback_timeframe, + datetime.now(timezone.utc) + timedelta( + minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min) + ).timestamp()) * 1000 - to_ms = int(arrow.utcnow() - .floor('minute') - .shift(minutes=-self._tf_in_min) - .int_timestamp) * 1000 + to_ms = int(timeframe_to_prev_date( + self._lookback_timeframe, + datetime.now(timezone.utc) - timedelta(minutes=self._tf_in_min) + ).timestamp()) * 1000 # todo: utc date output for starting date self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: " @@ -193,7 +193,7 @@ class VolumePairList(IPairList): ) in candles else None # in case of candle data calculate typical price and quoteVolume for candle if pair_candles is not None and not pair_candles.empty: - if self._exchange._ft_has["ohlcv_volume_currency"] == "base": + if self._exchange.get_option("ohlcv_volume_currency") == "base": pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3 diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 1de27fcbd..0cec734fb 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -1,5 +1,5 @@ import re -from typing import List +from typing import Any, Dict, List def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], @@ -40,3 +40,13 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], except re.error as err: raise ValueError(f"Wildcard error in {pair_wc}, {err}") return result + + +def dynamic_expand_pairlist(config: Dict[str, Any], markets: List[str]) -> List[str]: + expanded_pairs = expand_pairlist(config['pairs'], markets) + if config.get('freqai', {}).get('enabled', False): + corr_pairlist = config['freqai']['feature_parameters']['include_corr_pairlist'] + expanded_pairs += [pair for pair in corr_pairlist + if pair not in config['pairs']] + + return expanded_pairs diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index abc90a685..e80d13e9d 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -49,7 +49,7 @@ class StoplossGuard(IProtection): trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) trades = [trade for trade in trades1 if (str(trade.exit_reason) in ( ExitType.TRAILING_STOP_LOSS.value, ExitType.STOP_LOSS.value, - ExitType.STOPLOSS_ON_EXCHANGE.value) + ExitType.STOPLOSS_ON_EXCHANGE.value, ExitType.LIQUIDATION.value) and trade.close_profit and trade.close_profit < self._profit_limit)] if self._only_per_side: diff --git a/freqtrade/resolvers/freqaimodel_resolver.py b/freqtrade/resolvers/freqaimodel_resolver.py new file mode 100644 index 000000000..5a847bb2b --- /dev/null +++ b/freqtrade/resolvers/freqaimodel_resolver.py @@ -0,0 +1,57 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load a custom model for freqai +""" +import logging +from pathlib import Path +from typing import Dict + +from freqtrade.constants import USERPATH_FREQAIMODELS +from freqtrade.exceptions import OperationalException +from freqtrade.freqai.freqai_interface import IFreqaiModel +from freqtrade.resolvers import IResolver + + +logger = logging.getLogger(__name__) + + +class FreqaiModelResolver(IResolver): + """ + This class contains all the logic to load custom hyperopt loss class + """ + + object_type = IFreqaiModel + object_type_str = "FreqaiModel" + user_subdir = USERPATH_FREQAIMODELS + initial_search_path = ( + Path(__file__).parent.parent.joinpath("freqai/prediction_models").resolve() + ) + + @staticmethod + def load_freqaimodel(config: Dict) -> IFreqaiModel: + """ + Load the custom class from config parameter + :param config: configuration dictionary + """ + disallowed_models = ["BaseRegressionModel", "BaseTensorFlowModel"] + + freqaimodel_name = config.get("freqaimodel") + if not freqaimodel_name: + raise OperationalException( + "No freqaimodel set. Please use `--freqaimodel` to " + "specify the FreqaiModel class to use.\n" + ) + if freqaimodel_name in disallowed_models: + raise OperationalException( + f"{freqaimodel_name} is a baseclass and cannot be used directly. Please choose " + "an existing child class or inherit from this baseclass.\n" + ) + freqaimodel = FreqaiModelResolver.load_object( + freqaimodel_name, + config, + kwargs={"config": config}, + extra_dir=config.get("freqaimodel_path"), + ) + + return freqaimodel diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 74b28dffe..b99e7a94b 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -193,7 +193,10 @@ class IResolver: :return: List of dicts containing 'name', 'class' and 'location' entries """ logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'") - objects = [] + objects: List[Dict[str, Any]] = [] + if not directory.is_dir(): + logger.info(f"'{directory}' is not a directory, skipping.") + return objects for entry in directory.iterdir(): if ( recursive and entry.is_dir() diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 333f2fe6e..ada20230a 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -194,11 +194,11 @@ class OrderSchema(BaseModel): pair: str order_id: str status: str - remaining: float + remaining: Optional[float] amount: float safe_price: float cost: float - filled: float + filled: Optional[float] ft_order_side: str order_type: str is_open: bool @@ -325,11 +325,13 @@ class ForceEnterPayload(BaseModel): ordertype: Optional[OrderTypeValues] stakeamount: Optional[float] entry_tag: Optional[str] + leverage: Optional[float] class ForceExitPayload(BaseModel): tradeid: str ordertype: Optional[OrderTypeValues] + amount: Optional[float] class BlacklistPayload(BaseModel): diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index b3506409d..bf21715b7 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -37,7 +37,8 @@ logger = logging.getLogger(__name__) # 2.14: Add entry/exit orders to trade response # 2.15: Add backtest history endpoints # 2.16: Additional daily metrics -API_VERSION = 2.16 +# 2.17: Forceentry - leverage, partial force_exit +API_VERSION = 2.17 # Public API, requires no auth. router_public = APIRouter() @@ -142,12 +143,11 @@ def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(g @router.post('/forcebuy', response_model=ForceEnterResponse, tags=['trading']) def force_entry(payload: ForceEnterPayload, rpc: RPC = Depends(get_rpc)): ordertype = payload.ordertype.value if payload.ordertype else None - stake_amount = payload.stakeamount if payload.stakeamount else None - entry_tag = payload.entry_tag if payload.entry_tag else 'force_entry' trade = rpc._rpc_force_entry(payload.pair, payload.price, order_side=payload.side, - order_type=ordertype, stake_amount=stake_amount, - enter_tag=entry_tag) + order_type=ordertype, stake_amount=payload.stakeamount, + enter_tag=payload.entry_tag or 'force_entry', + leverage=payload.leverage) if trade: return ForceEnterResponse.parse_obj(trade.to_json()) @@ -161,7 +161,7 @@ def force_entry(payload: ForceEnterPayload, rpc: RPC = Depends(get_rpc)): @router.post('/forcesell', response_model=ResultMsg, tags=['trading']) def forceexit(payload: ForceExitPayload, rpc: RPC = Depends(get_rpc)): ordertype = payload.ordertype.value if payload.ordertype else None - return rpc._rpc_force_exit(payload.tradeid, ordertype) + return rpc._rpc_force_exit(payload.tradeid, ordertype, amount=payload.amount) @router.get('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist']) @@ -216,9 +216,10 @@ def stop(rpc: RPC = Depends(get_rpc)): return rpc._rpc_stop() +@router.post('/stopentry', response_model=StatusMsg, tags=['botcontrol']) @router.post('/stopbuy', response_model=StatusMsg, tags=['botcontrol']) def stop_buy(rpc: RPC = Depends(get_rpc)): - return rpc._rpc_stopbuy() + return rpc._rpc_stopentry() @router.post('/reload_config', response_model=StatusMsg, tags=['botcontrol']) diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index 81c013efa..66654c0b1 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -18,9 +18,9 @@ def get_rpc_optional() -> Optional[RPC]: def get_rpc() -> Optional[Iterator[RPC]]: _rpc = get_rpc_optional() if _rpc: - Trade.query.session.rollback() + Trade.rollback() yield _rpc - Trade.query.session.rollback() + Trade.rollback() else: raise RPCException('Bot is not in the correct state') diff --git a/freqtrade/rpc/api_server/web_ui.py b/freqtrade/rpc/api_server/web_ui.py index b04269c61..e1a277b30 100644 --- a/freqtrade/rpc/api_server/web_ui.py +++ b/freqtrade/rpc/api_server/web_ui.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Optional from fastapi import APIRouter from fastapi.exceptions import HTTPException @@ -50,8 +51,12 @@ async def index_html(rest_of_path: str): filename = uibase / rest_of_path # It's security relevant to check "relative_to". # Without this, Directory-traversal is possible. + media_type: Optional[str] = None + if filename.suffix == '.js': + # Force text/javascript for .js files - Circumvent faulty system configuration + media_type = 'application/javascript' if filename.is_file() and is_relative_to(filename, uibase): - return FileResponse(str(filename)) + return FileResponse(str(filename), media_type=media_type) index_file = uibase / 'index.html' if not index_file.is_file(): diff --git a/freqtrade/rpc/fiat_convert.py b/freqtrade/rpc/fiat_convert.py index df33693ac..cbe4c0045 100644 --- a/freqtrade/rpc/fiat_convert.py +++ b/freqtrade/rpc/fiat_convert.py @@ -12,6 +12,7 @@ from pycoingecko import CoinGeckoAPI from requests.exceptions import RequestException from freqtrade.constants import SUPPORTED_FIAT +from freqtrade.mixins.logging_mixin import LoggingMixin logger = logging.getLogger(__name__) @@ -27,7 +28,7 @@ coingecko_mapping = { } -class CryptoToFiatConverter: +class CryptoToFiatConverter(LoggingMixin): """ Main class to initiate Crypto to FIAT. This object contains a list of pair Crypto, FIAT @@ -54,6 +55,7 @@ class CryptoToFiatConverter: # Timeout: 6h self._pair_price: TTLCache = TTLCache(maxsize=500, ttl=6 * 60 * 60) + LoggingMixin.__init__(self, logger, 3600) self._load_cryptomap() def _load_cryptomap(self) -> None: @@ -177,7 +179,9 @@ class CryptoToFiatConverter: if not _gekko_id: # return 0 for unsupported stake currencies (fiat-convert should not break the bot) - logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol) + self.log_once( + f"unsupported crypto-symbol {crypto_symbol.upper()} - returning 0.0", + logger.warning) return 0.0 try: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index e6948c9e2..11311f671 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -179,8 +179,10 @@ class RPC: else: current_rate = trade.close_rate if len(trade.select_filled_orders(trade.entry_side)) > 0: - current_profit = trade.calc_profit_ratio(current_rate) - current_profit_abs = trade.calc_profit(current_rate) + current_profit = trade.calc_profit_ratio( + current_rate) if not isnan(current_rate) else NAN + current_profit_abs = trade.calc_profit( + current_rate) if not isnan(current_rate) else NAN current_profit_fiat: Optional[float] = None # Calculate fiat profit if self._fiat_converter: @@ -201,7 +203,7 @@ class RPC: trade_dict = trade.to_json() trade_dict.update(dict( - close_profit=trade.close_profit if trade.close_profit is not None else None, + close_profit=trade.close_profit if not trade.is_open else None, current_rate=current_rate, current_profit=current_profit, # Deprecated current_profit_pct=round(current_profit * 100, 2), # Deprecated @@ -239,12 +241,15 @@ class RPC: trade.pair, side='exit', is_short=trade.is_short, refresh=False) except (PricingError, ExchangeError): current_rate = NAN - if len(trade.select_filled_orders(trade.entry_side)) > 0: - trade_profit = trade.calc_profit(current_rate) - profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}' + trade_profit = NAN + profit_str = f'{NAN:.2%}' else: - trade_profit = 0.0 - profit_str = f'{0.0:.2f}' + if trade.nr_of_successful_entries > 0: + trade_profit = trade.calc_profit(current_rate) + profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}' + else: + trade_profit = 0.0 + profit_str = f'{0.0:.2f}' direction_str = ('S' if trade.is_short else 'L') if nonspot else '' if self._fiat_converter: fiat_profit = self._fiat_converter.convert_amount( @@ -424,21 +429,20 @@ class RPC: for trade in trades: current_rate: float = 0.0 - if not trade.open_rate: - continue if trade.close_date: durations.append((trade.close_date - trade.open_date).total_seconds()) if not trade.is_open: profit_ratio = trade.close_profit - profit_closed_coin.append(trade.close_profit_abs) + profit_abs = trade.close_profit_abs + profit_closed_coin.append(profit_abs) profit_closed_ratio.append(profit_ratio) if trade.close_profit >= 0: winning_trades += 1 - winning_profit += trade.close_profit_abs + winning_profit += profit_abs else: losing_trades += 1 - losing_profit += trade.close_profit_abs + losing_profit += profit_abs else: # Get current rate try: @@ -446,11 +450,15 @@ class RPC: trade.pair, side='exit', is_short=trade.is_short, refresh=False) except (PricingError, ExchangeError): current_rate = NAN - profit_ratio = trade.calc_profit_ratio(rate=current_rate) + if isnan(current_rate): + profit_ratio = NAN + profit_abs = NAN + else: + profit_ratio = trade.calc_profit_ratio(rate=current_rate) + profit_abs = trade.calc_profit( + rate=trade.close_rate or current_rate) + trade.realized_profit - profit_all_coin.append( - trade.calc_profit(rate=trade.close_rate or current_rate) - ) + profit_all_coin.append(profit_abs) profit_all_ratio.append(profit_ratio) best_pair = Trade.get_best_pair(start_date) @@ -649,7 +657,7 @@ class RPC: self._freqtrade.state = State.RELOAD_CONFIG return {'status': 'Reloading config ...'} - def _rpc_stopbuy(self) -> Dict[str, str]: + def _rpc_stopentry(self) -> Dict[str, str]: """ Handler to stop buying, but handle open trades gracefully. """ @@ -657,38 +665,50 @@ class RPC: # Set 'max_open_trades' to 0 self._freqtrade.config['max_open_trades'] = 0 - return {'status': 'No more buy will occur from now. Run /reload_config to reset.'} + return {'status': 'No more entries will occur from now. Run /reload_config to reset.'} - def _rpc_force_exit(self, trade_id: str, ordertype: Optional[str] = None) -> Dict[str, str]: + def __exec_force_exit(self, trade: Trade, ordertype: Optional[str], + amount: Optional[float] = None) -> None: + # Check if there is there is an open order + fully_canceled = False + if trade.open_order_id: + order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) + + if order['side'] == trade.entry_side: + fully_canceled = self._freqtrade.handle_cancel_enter( + trade, order, CANCEL_REASON['FORCE_EXIT']) + + if order['side'] == trade.exit_side: + # Cancel order - so it is placed anew with a fresh price. + self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_EXIT']) + + if not fully_canceled: + # Get current rate and execute sell + current_rate = self._freqtrade.exchange.get_rate( + trade.pair, side='exit', is_short=trade.is_short, refresh=True) + exit_check = ExitCheckTuple(exit_type=ExitType.FORCE_EXIT) + order_type = ordertype or self._freqtrade.strategy.order_types.get( + "force_exit", self._freqtrade.strategy.order_types["exit"]) + sub_amount: Optional[float] = None + if amount and amount < trade.amount: + # Partial exit ... + min_exit_stake = self._freqtrade.exchange.get_min_pair_stake_amount( + trade.pair, current_rate, trade.stop_loss_pct) + remaining = (trade.amount - amount) * current_rate + if remaining < min_exit_stake: + raise RPCException(f'Remaining amount of {remaining} would be too small.') + sub_amount = amount + + self._freqtrade.execute_trade_exit( + trade, current_rate, exit_check, ordertype=order_type, + sub_trade_amt=sub_amount) + + def _rpc_force_exit(self, trade_id: str, ordertype: Optional[str] = None, *, + amount: Optional[float] = None) -> Dict[str, str]: """ Handler for forceexit . Sells the given trade at current price """ - def _exec_force_exit(trade: Trade) -> None: - # Check if there is there is an open order - fully_canceled = False - if trade.open_order_id: - order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) - - if order['side'] == trade.entry_side: - fully_canceled = self._freqtrade.handle_cancel_enter( - trade, order, CANCEL_REASON['FORCE_EXIT']) - - if order['side'] == trade.exit_side: - # Cancel order - so it is placed anew with a fresh price. - self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_EXIT']) - - if not fully_canceled: - # Get current rate and execute sell - current_rate = self._freqtrade.exchange.get_rate( - trade.pair, side='exit', is_short=trade.is_short, refresh=True) - exit_check = ExitCheckTuple(exit_type=ExitType.FORCE_EXIT) - order_type = ordertype or self._freqtrade.strategy.order_types.get( - "force_exit", self._freqtrade.strategy.order_types["exit"]) - - self._freqtrade.execute_trade_exit( - trade, current_rate, exit_check, ordertype=order_type) - # ---- EOF def _exec_forcesell ---- if self._freqtrade.state != State.RUNNING: raise RPCException('trader is not running') @@ -697,7 +717,7 @@ class RPC: if trade_id == 'all': # Execute sell for all open orders for trade in Trade.get_open_trades(): - _exec_force_exit(trade) + self.__exec_force_exit(trade, ordertype) Trade.commit() self._freqtrade.wallets.update() return {'result': 'Created sell orders for all open trades.'} @@ -710,7 +730,7 @@ class RPC: logger.warning('force_exit: Invalid argument received') raise RPCException('invalid argument') - _exec_force_exit(trade) + self.__exec_force_exit(trade, ordertype, amount) Trade.commit() self._freqtrade.wallets.update() return {'result': f'Created sell order for trade {trade_id}.'} @@ -719,7 +739,8 @@ class RPC: order_type: Optional[str] = None, order_side: SignalDirection = SignalDirection.LONG, stake_amount: Optional[float] = None, - enter_tag: Optional[str] = 'force_entry') -> Optional[Trade]: + enter_tag: Optional[str] = 'force_entry', + leverage: Optional[float] = None) -> Optional[Trade]: """ Handler for forcebuy Buys a pair trade at the given or current price @@ -761,6 +782,7 @@ class RPC: ordertype=order_type, trade=trade, is_short=is_short, enter_tag=enter_tag, + leverage_=leverage, ): Trade.commit() trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first() @@ -875,7 +897,7 @@ class RPC: lock.active = False lock.lock_end_time = datetime.now(timezone.utc) - PairLock.query.session.commit() + Trade.commit() return self._rpc_locks() diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 66e84029f..3ccf23228 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -2,6 +2,7 @@ This module contains class to manage RPC communications (Telegram, API, ...) """ import logging +from collections import deque from typing import Any, Dict, List from freqtrade.enums import RPCMessageType @@ -77,6 +78,17 @@ class RPCManager: except NotImplementedError: logger.error(f"Message type '{msg['type']}' not implemented by handler {mod.name}.") + def process_msg_queue(self, queue: deque) -> None: + """ + Process all messages in the queue. + """ + while queue: + msg = queue.popleft() + self.send_msg({ + 'type': RPCMessageType.STRATEGY_MSG, + 'msg': msg, + }) + def startup_messages(self, config: Dict[str, Any], pairlist, protections) -> None: if config['dry_run']: self.send_msg({ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 2aff1d210..8c988d570 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -16,8 +16,8 @@ from typing import Any, Callable, Dict, List, Optional, Union import arrow from tabulate import tabulate -from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, - ParseMode, ReplyKeyboardMarkup, Update) +from telegram import (MAX_MESSAGE_LENGTH, CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, + KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update) from telegram.error import BadRequest, NetworkError, TelegramError from telegram.ext import CallbackContext, CallbackQueryHandler, CommandHandler, Updater from telegram.utils.helpers import escape_markdown @@ -35,8 +35,6 @@ logger = logging.getLogger(__name__) logger.debug('Included module rpc.telegram ...') -MAX_TELEGRAM_MESSAGE_LENGTH = 4096 - @dataclass class TimeunitMappings: @@ -72,7 +70,7 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: ) return wrapper # Rollback session to avoid getting data stored in a transaction. - Trade.query.session.rollback() + Trade.rollback() logger.debug( 'Executing handler: %s for chat_id: %s', command_handler.__name__, @@ -116,17 +114,20 @@ class Telegram(RPCHandler): # TODO: DRY! - its not good to list all valid cmds here. But otherwise # this needs refactoring of the whole telegram module (same # problem in _help()). - valid_keys: List[str] = [r'/start$', r'/stop$', r'/status$', r'/status table$', - r'/trades$', r'/performance$', r'/buys', r'/entries', - r'/sells', r'/exits', r'/mix_tags', - r'/daily$', r'/daily \d+$', r'/profit$', r'/profit \d+', - r'/stats$', r'/count$', r'/locks$', r'/balance$', - r'/stopbuy$', r'/reload_config$', r'/show_config$', - r'/logs$', r'/whitelist$', r'/blacklist$', r'/bl_delete$', - r'/weekly$', r'/weekly \d+$', r'/monthly$', r'/monthly \d+$', - r'/forcebuy$', r'/forcelong$', r'/forceshort$', - r'/forcesell$', r'/forceexit$', - r'/edge$', r'/health$', r'/help$', r'/version$'] + valid_keys: List[str] = [ + r'/start$', r'/stop$', r'/status$', r'/status table$', + r'/trades$', r'/performance$', r'/buys', r'/entries', + r'/sells', r'/exits', r'/mix_tags', + r'/daily$', r'/daily \d+$', r'/profit$', r'/profit \d+', + r'/stats$', r'/count$', r'/locks$', r'/balance$', + r'/stopbuy$', r'/stopentry$', r'/reload_config$', r'/show_config$', + r'/logs$', r'/whitelist$', r'/whitelist(\ssorted|\sbaseonly)+$', + r'/blacklist$', r'/bl_delete$', + r'/weekly$', r'/weekly \d+$', r'/monthly$', r'/monthly \d+$', + r'/forcebuy$', r'/forcelong$', r'/forceshort$', + r'/forcesell$', r'/forceexit$', + r'/edge$', r'/health$', r'/help$', r'/version$' + ] # Create keys for generation valid_keys_print = [k.replace('$', '') for k in valid_keys] @@ -183,7 +184,7 @@ class Telegram(RPCHandler): CommandHandler(['unlock', 'delete_locks'], self._delete_locks), CommandHandler(['reload_config', 'reload_conf'], self._reload_config), CommandHandler(['show_config', 'show_conf'], self._show_config), - CommandHandler('stopbuy', self._stopbuy), + CommandHandler(['stopbuy', 'stopentry'], self._stopentry), CommandHandler('whitelist', self._whitelist), CommandHandler('blacklist', self._blacklist), CommandHandler(['blacklist_delete', 'bl_delete'], self._blacklist_delete), @@ -274,7 +275,7 @@ class Telegram(RPCHandler): f"{emoji} *{self._exchange_from_msg(msg)}:*" f" {entry_side['entered'] if is_fill else entry_side['enter']} {msg['pair']}" f" (#{msg['trade_id']})\n" - ) + ) message += self._add_analyzed_candle(msg['pair']) message += f"*Enter Tag:* `{msg['enter_tag']}`\n" if msg.get('enter_tag') else "" message += f"*Amount:* `{msg['amount']:.8f}`\n" @@ -315,20 +316,36 @@ class Telegram(RPCHandler): msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount( msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) msg['profit_extra'] = ( - f" ({msg['gain']}: {msg['profit_amount']:.8f} {msg['stake_currency']}" - f" / {msg['profit_fiat']:.3f} {msg['fiat_currency']})") + f" / {msg['profit_fiat']:.3f} {msg['fiat_currency']}") else: msg['profit_extra'] = '' + msg['profit_extra'] = ( + f" ({msg['gain']}: {msg['profit_amount']:.8f} {msg['stake_currency']}" + f"{msg['profit_extra']})") is_fill = msg['type'] == RPCMessageType.EXIT_FILL + is_sub_trade = msg.get('sub_trade') + is_sub_profit = msg['profit_amount'] != msg.get('cumulative_profit') + profit_prefix = ('Sub ' if is_sub_profit + else 'Cumulative ') if is_sub_trade else '' + cp_extra = '' + if is_sub_profit and is_sub_trade: + if self._rpc._fiat_converter: + cp_fiat = self._rpc._fiat_converter.convert_amount( + msg['cumulative_profit'], msg['stake_currency'], msg['fiat_currency']) + cp_extra = f" / {cp_fiat:.3f} {msg['fiat_currency']}" + else: + cp_extra = '' + cp_extra = f"*Cumulative Profit:* (`{msg['cumulative_profit']:.8f} " \ + f"{msg['stake_currency']}{cp_extra}`)\n" message = ( f"{msg['emoji']} *{self._exchange_from_msg(msg)}:* " f"{'Exited' if is_fill else 'Exiting'} {msg['pair']} (#{msg['trade_id']})\n" f"{self._add_analyzed_candle(msg['pair'])}" - f"*{'Profit' if is_fill else 'Unrealized Profit'}:* " + f"*{f'{profit_prefix}Profit' if is_fill else f'Unrealized {profit_prefix}Profit'}:* " f"`{msg['profit_ratio']:.2%}{msg['profit_extra']}`\n" + f"{cp_extra}" f"*Enter Tag:* `{msg['enter_tag']}`\n" f"*Exit Reason:* `{msg['exit_reason']}`\n" - f"*Duration:* `{msg['duration']} ({msg['duration_min']:.1f} min)`\n" f"*Direction:* `{msg['direction']}`\n" f"{msg['leverage_text']}" f"*Amount:* `{msg['amount']:.8f}`\n" @@ -336,11 +353,25 @@ class Telegram(RPCHandler): ) if msg['type'] == RPCMessageType.EXIT: message += (f"*Current Rate:* `{msg['current_rate']:.8f}`\n" - f"*Close Rate:* `{msg['limit']:.8f}`") + f"*Exit Rate:* `{msg['limit']:.8f}`") elif msg['type'] == RPCMessageType.EXIT_FILL: - message += f"*Close Rate:* `{msg['close_rate']:.8f}`" + message += f"*Exit Rate:* `{msg['close_rate']:.8f}`" + if msg.get('sub_trade'): + if self._rpc._fiat_converter: + msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount( + msg['stake_amount'], msg['stake_currency'], msg['fiat_currency']) + else: + msg['stake_amount_fiat'] = 0 + rem = round_coin_value(msg['stake_amount'], msg['stake_currency']) + message += f"\n*Remaining:* `({rem}" + if msg.get('fiat_currency', None): + message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}" + + message += ")`" + else: + message += f"\n*Duration:* `{msg['duration']} ({msg['duration_min']:.1f} min)`" return message def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> str: @@ -353,7 +384,8 @@ class Telegram(RPCHandler): elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL): msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit' message = (f"\N{WARNING SIGN} *{self._exchange_from_msg(msg)}:* " - f"Cancelling {msg['message_side']} Order for {msg['pair']} " + f"Cancelling {'partial ' if msg.get('sub_trade') else ''}" + f"{msg['message_side']} Order for {msg['pair']} " f"(#{msg['trade_id']}). Reason: {msg['reason']}.") elif msg_type == RPCMessageType.PROTECTION_TRIGGER: @@ -376,7 +408,8 @@ class Telegram(RPCHandler): elif msg_type == RPCMessageType.STARTUP: message = f"{msg['status']}" - + elif msg_type == RPCMessageType.STRATEGY_MSG: + message = f"{msg['msg']}" else: raise NotImplementedError(f"Unknown message type: {msg_type}") return message @@ -423,54 +456,63 @@ class Telegram(RPCHandler): else: return "\N{CROSS MARK}" - def _prepare_entry_details(self, filled_orders: List, quote_currency: str, is_open: bool): + def _prepare_order_details(self, filled_orders: List, quote_currency: str, is_open: bool): """ Prepare details of trade with entry adjustment enabled """ - lines: List[str] = [] + lines_detail: List[str] = [] if len(filled_orders) > 0: first_avg = filled_orders[0]["safe_price"] for x, order in enumerate(filled_orders): - if not order['ft_is_entry'] or order['is_open'] is True: + lines: List[str] = [] + if order['is_open'] is True: continue + wording = 'Entry' if order['ft_is_entry'] else 'Exit' + cur_entry_datetime = arrow.get(order["order_filled_date"]) - cur_entry_amount = order["amount"] + cur_entry_amount = order["filled"] or order["amount"] cur_entry_average = order["safe_price"] lines.append(" ") if x == 0: - lines.append(f"*Entry #{x+1}:*") + lines.append(f"*{wording} #{x+1}:*") lines.append( - f"*Entry Amount:* {cur_entry_amount} ({order['cost']:.8f} {quote_currency})") - lines.append(f"*Average Entry Price:* {cur_entry_average}") + f"*Amount:* {cur_entry_amount} ({order['cost']:.8f} {quote_currency})") + lines.append(f"*Average Price:* {cur_entry_average}") else: sumA = 0 sumB = 0 for y in range(x): - sumA += (filled_orders[y]["amount"] * filled_orders[y]["safe_price"]) - sumB += filled_orders[y]["amount"] + amount = filled_orders[y]["filled"] or filled_orders[y]["amount"] + sumA += amount * filled_orders[y]["safe_price"] + sumB += amount prev_avg_price = sumA / sumB + # TODO: This calculation ignores fees. price_to_1st_entry = ((cur_entry_average - first_avg) / first_avg) minus_on_entry = 0 if prev_avg_price: minus_on_entry = (cur_entry_average - prev_avg_price) / prev_avg_price - dur_entry = cur_entry_datetime - arrow.get( - filled_orders[x - 1]["order_filled_date"]) - days = dur_entry.days - hours, remainder = divmod(dur_entry.seconds, 3600) - minutes, seconds = divmod(remainder, 60) - lines.append(f"*Entry #{x+1}:* at {minus_on_entry:.2%} avg profit") + lines.append(f"*{wording} #{x+1}:* at {minus_on_entry:.2%} avg profit") if is_open: lines.append("({})".format(cur_entry_datetime .humanize(granularity=["day", "hour", "minute"]))) lines.append( - f"*Entry Amount:* {cur_entry_amount} ({order['cost']:.8f} {quote_currency})") - lines.append(f"*Average Entry Price:* {cur_entry_average} " + f"*Amount:* {cur_entry_amount} ({order['cost']:.8f} {quote_currency})") + lines.append(f"*Average {wording} Price:* {cur_entry_average} " f"({price_to_1st_entry:.2%} from 1st entry rate)") - lines.append(f"*Order filled at:* {order['order_filled_date']}") - lines.append(f"({days}d {hours}h {minutes}m {seconds}s from previous entry)") - return lines + lines.append(f"*Order filled:* {order['order_filled_date']}") + + # TODO: is this really useful? + # dur_entry = cur_entry_datetime - arrow.get( + # filled_orders[x - 1]["order_filled_date"]) + # days = dur_entry.days + # hours, remainder = divmod(dur_entry.seconds, 3600) + # minutes, seconds = divmod(remainder, 60) + # lines.append( + # f"({days}d {hours}h {minutes}m {seconds}s from previous {wording.lower()})") + lines_detail.append("\n".join(lines)) + return lines_detail @authorized_only def _status(self, update: Update, context: CallbackContext) -> None: @@ -485,7 +527,14 @@ class Telegram(RPCHandler): if context.args and 'table' in context.args: self._status_table(update, context) return + else: + self._status_msg(update, context) + def _status_msg(self, update: Update, context: CallbackContext) -> None: + """ + handler for `/status` and `/status `. + + """ try: # Check if there's at least one numerical ID provided. @@ -497,14 +546,13 @@ class Telegram(RPCHandler): results = self._rpc._rpc_trade_status(trade_ids=trade_ids) position_adjust = self._config.get('position_adjustment_enable', False) max_entries = self._config.get('max_entry_position_adjustment', -1) - messages = [] for r in results: r['open_date_hum'] = arrow.get(r['open_date']).humanize() r['num_entries'] = len([o for o in r['orders'] if o['ft_is_entry']]) r['exit_reason'] = r.get('exit_reason', "") lines = [ "*Trade ID:* `{trade_id}`" + - ("` (since {open_date_hum})`" if r['is_open'] else ""), + (" `(since {open_date_hum})`" if r['is_open'] else ""), "*Current Pair:* {pair}", "*Direction:* " + ("`Short`" if r.get('is_short') else "`Long`"), "*Leverage:* `{leverage}`" if r.get('leverage') else "", @@ -528,6 +576,8 @@ class Telegram(RPCHandler): ]) if r['is_open']: + if r.get('realized_profit'): + lines.append("*Realized Profit:* `{realized_profit:.8f}`") if (r['stop_loss_abs'] != r['initial_stop_loss_abs'] and r['initial_stop_loss_ratio'] is not None): # Adding initial stoploss only if it is different from stoploss @@ -540,24 +590,34 @@ class Telegram(RPCHandler): lines.append("*Stoploss distance:* `{stoploss_current_dist:.8f}` " "`({stoploss_current_dist_ratio:.2%})`") if r['open_order']: - if r['exit_order_status']: - lines.append("*Open Order:* `{open_order}` - `{exit_order_status}`") - else: - lines.append("*Open Order:* `{open_order}`") + lines.append( + "*Open Order:* `{open_order}`" + + "- `{exit_order_status}`" if r['exit_order_status'] else "") - lines_detail = self._prepare_entry_details( + lines_detail = self._prepare_order_details( r['orders'], r['quote_currency'], r['is_open']) lines.extend(lines_detail if lines_detail else "") - - # Filter empty lines using list-comprehension - messages.append("\n".join([line for line in lines if line]).format(**r)) - - for msg in messages: - self._send_msg(msg) + self.__send_status_msg(lines, r) except RPCException as e: self._send_msg(str(e)) + def __send_status_msg(self, lines: List[str], r: Dict[str, Any]) -> None: + """ + Send status message. + """ + msg = '' + + for line in lines: + if line: + if (len(msg) + len(line) + 1) < MAX_MESSAGE_LENGTH: + msg += line + '\n' + else: + self._send_msg(msg.format(**r)) + msg = "*Trade ID:* `{trade_id}` - continued\n" + line + '\n' + + self._send_msg(msg.format(**r)) + @authorized_only def _status_table(self, update: Update, context: CallbackContext) -> None: """ @@ -860,7 +920,7 @@ class Telegram(RPCHandler): total_dust_currencies += 1 # Handle overflowing message length - if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH: + if len(output + curr_output) >= MAX_MESSAGE_LENGTH: self._send_msg(output) output = curr_output else: @@ -926,7 +986,7 @@ class Telegram(RPCHandler): self._send_msg(f"Status: `{msg['status']}`") @authorized_only - def _stopbuy(self, update: Update, context: CallbackContext) -> None: + def _stopentry(self, update: Update, context: CallbackContext) -> None: """ Handler for /stop_buy. Sets max_open_trades to 0 and gracefully sells all open trades @@ -934,7 +994,7 @@ class Telegram(RPCHandler): :param update: message update :return: None """ - msg = self._rpc._rpc_stopbuy() + msg = self._rpc._rpc_stopentry() self._send_msg(f"Status: `{msg['status']}`") @authorized_only @@ -1123,7 +1183,7 @@ class Telegram(RPCHandler): f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") - if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH: + if len(output + stat_line) >= MAX_MESSAGE_LENGTH: self._send_msg(output, parse_mode=ParseMode.HTML) output = stat_line else: @@ -1158,7 +1218,7 @@ class Telegram(RPCHandler): f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") - if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH: + if len(output + stat_line) >= MAX_MESSAGE_LENGTH: self._send_msg(output, parse_mode=ParseMode.HTML) output = stat_line else: @@ -1193,7 +1253,7 @@ class Telegram(RPCHandler): f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") - if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH: + if len(output + stat_line) >= MAX_MESSAGE_LENGTH: self._send_msg(output, parse_mode=ParseMode.HTML) output = stat_line else: @@ -1228,7 +1288,7 @@ class Telegram(RPCHandler): f"({trade['profit']:.2%}) " f"({trade['count']})\n") - if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH: + if len(output + stat_line) >= MAX_MESSAGE_LENGTH: self._send_msg(output, parse_mode=ParseMode.HTML) output = stat_line else: @@ -1311,6 +1371,12 @@ class Telegram(RPCHandler): try: whitelist = self._rpc._rpc_whitelist() + if context.args: + if "sorted" in context.args: + whitelist['whitelist'] = sorted(whitelist['whitelist']) + if "baseonly" in context.args: + whitelist['whitelist'] = [pair.split("/")[0] for pair in whitelist['whitelist']] + message = f"Using whitelist `{whitelist['method']}` with {whitelist['length']} pairs\n" message += f"`{', '.join(whitelist['whitelist'])}`" @@ -1367,7 +1433,7 @@ class Telegram(RPCHandler): escape_markdown(logrec[2], version=2), escape_markdown(logrec[3], version=2), escape_markdown(logrec[4], version=2)) - if len(msgs + msg) + 10 >= MAX_TELEGRAM_MESSAGE_LENGTH: + if len(msgs + msg) + 10 >= MAX_MESSAGE_LENGTH: # Send message immediately if it would become too long self._send_msg(msgs, parse_mode=ParseMode.MARKDOWN_V2) msgs = msg + '\n' @@ -1424,13 +1490,14 @@ class Telegram(RPCHandler): "------------\n" "*/start:* `Starts the trader`\n" "*/stop:* Stops the trader\n" - "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n" + "*/stopentry:* `Stops entering, but handles open trades gracefully` \n" "*/forceexit |all:* `Instantly exits the given trade or all trades, " "regardless of profit`\n" "*/fx |all:* `Alias to /forceexit`\n" f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}" "*/delete :* `Instantly delete the given trade in the database`\n" - "*/whitelist:* `Show current whitelist` \n" + "*/whitelist [sorted] [baseonly]:* `Show current whitelist. Optionally in " + "order and/or only displaying the base currency of each pairing.`\n" "*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs " "to the blacklist.` \n" "*/blacklist_delete [pairs]| /bl_delete [pairs]:* " @@ -1467,7 +1534,7 @@ class Telegram(RPCHandler): "*/weekly :* `Shows statistics per week, over the last n weeks`\n" "*/monthly :* `Shows statistics per month, over the last n months`\n" "*/stats:* `Shows Wins / losses by Sell reason as well as " - "Avg. holding durationsfor buys and sells.`\n" + "Avg. holding durations for buys and sells.`\n" "*/help:* `This help message`\n" "*/version:* `Show version`" ) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c60817c99..79dbd4c69 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -145,11 +145,29 @@ class IStrategy(ABC, HyperStrategyMixin): informative_data.candle_type = config['candle_type_def'] self._ft_informative.append((informative_data, cls_method)) + def load_freqAI_model(self) -> None: + if self.config.get('freqai', {}).get('enabled', False): + # Import here to avoid importing this if freqAI is disabled + from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver + + self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) + self.freqai_info = self.config["freqai"] + else: + # Gracious failures if freqAI is disabled but "start" is called. + class DummyClass(): + def start(self, *args, **kwargs): + raise OperationalException( + 'freqAI is not enabled. ' + 'Please enable it in your config to use this strategy.') + self.freqai = DummyClass() # type: ignore + def ft_bot_start(self, **kwargs) -> None: """ Strategy init - runs after dataprovider has been added. Must call bot_start() """ + self.load_freqAI_model() + strategy_safe_wrapper(self.bot_start)() self.ft_load_hyper_params(self.config.get('runmode') == RunMode.HYPEROPT) @@ -463,10 +481,13 @@ class IStrategy(ABC, HyperStrategyMixin): def adjust_trade_position(self, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, min_stake: Optional[float], max_stake: float, + current_entry_rate: float, current_exit_rate: float, + current_entry_profit: float, current_exit_profit: float, **kwargs) -> Optional[float]: """ - Custom trade adjustment logic, returning the stake amount that a trade should be increased. - This means extra buy orders with additional fees. + Custom trade adjustment logic, returning the stake amount that a trade should be + increased or decreased. + This means extra buy or sell orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -477,10 +498,16 @@ class IStrategy(ABC, HyperStrategyMixin): :param current_time: datetime object, containing the current datetime :param current_rate: Current buy rate. :param current_profit: Current profit (as ratio), calculated based on current_rate. - :param min_stake: Minimal stake size allowed by exchange. - :param max_stake: Balance available for trading. + :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) + :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). + :param current_entry_rate: Current rate using entry pricing. + :param current_exit_rate: Current rate using exit pricing. + :param current_entry_profit: Current profit using entry pricing. + :param current_exit_profit: Current profit using exit pricing. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: Stake amount to adjust your trade + :return float: Stake amount to adjust your trade, + Positive values to increase position, Negative values to decrease position. + Return None for no action. """ return None @@ -548,6 +575,22 @@ class IStrategy(ABC, HyperStrategyMixin): """ return None + def populate_any_indicators(self, pair: str, df: DataFrame, tf: str, + informative: DataFrame = None, + set_generalized_indicators: bool = False) -> DataFrame: + """ + Function designed to automatically generate, name and merge features + from user indicated timeframes in the configuration file. User can add + additional features here, but must follow the naming convention. + This method is *only* used in FreqaiDataKitchen class and therefore + it is only called if FreqAI is active. + :param pair: pair to be used as informative + :param df: strategy dataframe which will receive merges from informatives + :param tf: timeframe of the dataframe which will modify the feature names + :param informative: the dataframe associated with the informative pair + """ + return df + ### # END - Intended to be overridden by strategy ### @@ -574,9 +617,6 @@ class IStrategy(ABC, HyperStrategyMixin): ) informative_pairs.append(pair_tf) else: - if not self.dp: - raise OperationalException('@informative decorator with unspecified asset ' - 'requires DataProvider instance.') for pair in self.dp.current_whitelist(): informative_pairs.append((pair, inf_data.timeframe, candle_type)) return list(set(informative_pairs)) @@ -670,10 +710,9 @@ class IStrategy(ABC, HyperStrategyMixin): # Defs that only make change on new candle data. dataframe = self.analyze_ticker(dataframe, metadata) self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date'] - if self.dp: - self.dp._set_cached_df( - pair, self.timeframe, dataframe, - candle_type=self.config.get('candle_type_def', CandleType.SPOT)) + self.dp._set_cached_df( + pair, self.timeframe, dataframe, + candle_type=self.config.get('candle_type_def', CandleType.SPOT)) else: logger.debug("Skipping TA Analysis for already analyzed candle") dataframe[SignalType.ENTER_LONG.value] = 0 @@ -694,8 +733,6 @@ class IStrategy(ABC, HyperStrategyMixin): The analyzed dataframe is then accessible via `dp.get_analyzed_dataframe()`. :param pair: Pair to analyze. """ - if not self.dp: - raise OperationalException("DataProvider not found.") dataframe = self.dp.ohlcv( pair, self.timeframe, candle_type=self.config.get('candle_type_def', CandleType.SPOT) ) @@ -963,7 +1000,7 @@ class IStrategy(ABC, HyperStrategyMixin): # ROI # Trailing stoploss - if stoplossflag.exit_type == ExitType.STOP_LOSS: + if stoplossflag.exit_type in (ExitType.STOP_LOSS, ExitType.LIQUIDATION): logger.debug(f"{trade.pair} - Stoploss hit. exit_type={stoplossflag.exit_type}") exits.append(stoplossflag) @@ -1035,6 +1072,17 @@ class IStrategy(ABC, HyperStrategyMixin): sl_higher_long = (trade.stop_loss >= (low or current_rate) and not trade.is_short) sl_lower_short = (trade.stop_loss <= (high or current_rate) and trade.is_short) + liq_higher_long = (trade.liquidation_price + and trade.liquidation_price >= (low or current_rate) + and not trade.is_short) + liq_lower_short = (trade.liquidation_price + and trade.liquidation_price <= (high or current_rate) + and trade.is_short) + + if (liq_higher_long or liq_lower_short): + logger.debug(f"{trade.pair} - Liquidation price hit. exit_type=ExitType.LIQUIDATION") + return ExitCheckTuple(exit_type=ExitType.LIQUIDATION) + # evaluate if the stoploss was hit if stoploss is not on exchange # in Dry-Run, this handles stoploss logic as well, as the logic will not be different to # regular stoploss handling. @@ -1052,13 +1100,6 @@ class IStrategy(ABC, HyperStrategyMixin): f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at {trade.initial_stop_loss:.6f}, " f"trade opened at {trade.open_rate:.6f}") - new_stoploss = ( - trade.stop_loss + trade.initial_stop_loss - if trade.is_short else - trade.stop_loss - trade.initial_stop_loss - ) - logger.debug(f"{trade.pair} - Trailing stop saved " - f"{new_stoploss:.6f}") return ExitCheckTuple(exit_type=exit_type) diff --git a/freqtrade/strategy/parameters.py b/freqtrade/strategy/parameters.py index 83dd41de9..c6037ae0b 100644 --- a/freqtrade/strategy/parameters.py +++ b/freqtrade/strategy/parameters.py @@ -7,6 +7,9 @@ from abc import ABC, abstractmethod from contextlib import suppress from typing import Any, Optional, Sequence, Union +from freqtrade.enums.hyperoptstate import HyperoptState +from freqtrade.optimize.hyperopt_tools import HyperoptStateContainer + with suppress(ImportError): from skopt.space import Integer, Real, Categorical @@ -57,6 +60,13 @@ class BaseParameter(ABC): Get-space - will be used by Hyperopt to get the hyperopt Space """ + def can_optimize(self): + return ( + self.in_space + and self.optimize + and HyperoptStateContainer.state != HyperoptState.OPTIMIZE + ) + class NumericParameter(BaseParameter): """ Internal parameter used for Numeric purposes """ @@ -133,7 +143,7 @@ class IntParameter(NumericParameter): Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid calculating 100ds of indicators. """ - if self.in_space and self.optimize: + if self.can_optimize(): # Scikit-optimize ranges are "inclusive", while python's "range" is exclusive return range(self.low, self.high + 1) else: @@ -212,7 +222,7 @@ class DecimalParameter(NumericParameter): Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid calculating 100ds of indicators. """ - if self.in_space and self.optimize: + if self.can_optimize(): low = int(self.low * pow(10, self._decimals)) high = int(self.high * pow(10, self._decimals)) + 1 return [round(n * pow(0.1, self._decimals), self._decimals) for n in range(low, high)] @@ -261,7 +271,7 @@ class CategoricalParameter(BaseParameter): Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid calculating 100ds of indicators. """ - if self.in_space and self.optimize: + if self.can_optimize(): return self.opt_range else: return [self.value] diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py new file mode 100644 index 000000000..5810e7881 --- /dev/null +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -0,0 +1,328 @@ +import logging +from functools import reduce + +import pandas as pd +import talib.abstract as ta +from pandas import DataFrame +from technical import qtpylib + +from freqtrade.exchange import timeframe_to_prev_date +from freqtrade.persistence import Trade +from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, merge_informative_pair + + +logger = logging.getLogger(__name__) + + +class FreqaiExampleStrategy(IStrategy): + """ + Example strategy showing how the user connects their own + IFreqaiModel to the strategy. Namely, the user uses: + self.freqai.start(dataframe, metadata) + + to make predictions on their data. populate_any_indicators() automatically + generates the variety of features indicated by the user in the + canonical freqtrade configuration file under config['freqai']. + """ + + minimal_roi = {"0": 0.1, "240": -1} + + plot_config = { + "main_plot": {}, + "subplots": { + "prediction": {"prediction": {"color": "blue"}}, + "target_roi": { + "target_roi": {"color": "brown"}, + }, + "do_predict": { + "do_predict": {"color": "brown"}, + }, + }, + } + + process_only_new_candles = True + stoploss = -0.05 + use_exit_signal = True + startup_candle_count: int = 300 + can_short = False + + linear_roi_offset = DecimalParameter( + 0.00, 0.02, default=0.005, space="sell", optimize=False, load=True + ) + max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) + + def informative_pairs(self): + whitelist_pairs = self.dp.current_whitelist() + corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] + informative_pairs = [] + for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: + for pair in whitelist_pairs: + informative_pairs.append((pair, tf)) + for pair in corr_pairs: + if pair in whitelist_pairs: + continue # avoid duplication + informative_pairs.append((pair, tf)) + return informative_pairs + + def populate_any_indicators( + self, pair, df, tf, informative=None, set_generalized_indicators=False + ): + """ + Function designed to automatically generate, name and merge features + from user indicated timeframes in the configuration file. User controls the indicators + passed to the training/prediction by prepending indicators with `'%-' + coin ` + (see convention below). I.e. user should not prepend any supporting metrics + (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the + model. + :param pair: pair to be used as informative + :param df: strategy dataframe which will receive merges from informatives + :param tf: timeframe of the dataframe which will modify the feature names + :param informative: the dataframe associated with the informative pair + """ + + coin = pair.split('/')[0] + + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) + + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) + informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) + + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + + bollinger = qtpylib.bollinger_bands( + qtpylib.typical_price(informative), window=t, stds=2.2 + ) + informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] + informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] + informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] + + informative[f"%-{coin}bb_width-period_{t}"] = ( + informative[f"{coin}bb_upperband-period_{t}"] + - informative[f"{coin}bb_lowerband-period_{t}"] + ) / informative[f"{coin}bb_middleband-period_{t}"] + informative[f"%-{coin}close-bb_lower-period_{t}"] = ( + informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] + ) + + informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) + + informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative["volume"] / informative["volume"].rolling(t).mean() + ) + + informative[f"%-{coin}pct-change"] = informative["close"].pct_change() + informative[f"%-{coin}raw_volume"] = informative["volume"] + informative[f"%-{coin}raw_price"] = informative["close"] + + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) + + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) + + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + + # user adds targets here by prepending them with &- (see convention below) + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) + + # Classifiers are typically set up with strings as targets: + # df['&s-up_or_down'] = np.where( df["close"].shift(-100) > + # df["close"], 'up', 'down') + + # If user wishes to use multiple targets, they can add more by + # appending more columns with '&'. User should keep in mind that multi targets + # requires a multioutput prediction model such as + # templates/CatboostPredictionMultiModel.py, + + # df["&-s_range"] = ( + # df["close"] + # .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + # .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + # .max() + # - + # df["close"] + # .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + # .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + # .min() + # ) + + return df + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + # All indicators must be populated by populate_any_indicators() for live functionality + # to work correctly. + + # the model will return all labels created by user in `populate_any_indicators` + # (& appended targets), an indication of whether or not the prediction should be accepted, + # the target mean/std values for each of the labels created by user in + # `populate_any_indicators()` for each training period. + + dataframe = self.freqai.start(dataframe, metadata, self) + + dataframe["target_roi"] = dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * 1.25 + dataframe["sell_roi"] = dataframe["&-s_close_mean"] - dataframe["&-s_close_std"] * 1.25 + return dataframe + + def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame: + + enter_long_conditions = [df["do_predict"] == 1, df["&-s_close"] > df["target_roi"]] + + if enter_long_conditions: + df.loc[ + reduce(lambda x, y: x & y, enter_long_conditions), ["enter_long", "enter_tag"] + ] = (1, "long") + + enter_short_conditions = [df["do_predict"] == 1, df["&-s_close"] < df["sell_roi"]] + + if enter_short_conditions: + df.loc[ + reduce(lambda x, y: x & y, enter_short_conditions), ["enter_short", "enter_tag"] + ] = (1, "short") + + return df + + def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame: + exit_long_conditions = [df["do_predict"] == 1, df["&-s_close"] < df["sell_roi"] * 0.25] + if exit_long_conditions: + df.loc[reduce(lambda x, y: x & y, exit_long_conditions), "exit_long"] = 1 + + exit_short_conditions = [df["do_predict"] == 1, df["&-s_close"] > df["target_roi"] * 0.25] + if exit_short_conditions: + df.loc[reduce(lambda x, y: x & y, exit_short_conditions), "exit_short"] = 1 + + return df + + def get_ticker_indicator(self): + return int(self.config["timeframe"][:-1]) + + def custom_exit( + self, pair: str, trade: Trade, current_time, current_rate, current_profit, **kwargs + ): + + dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) + + trade_date = timeframe_to_prev_date(self.config["timeframe"], trade.open_date_utc) + trade_candle = dataframe.loc[(dataframe["date"] == trade_date)] + + if trade_candle.empty: + return None + trade_candle = trade_candle.squeeze() + + follow_mode = self.config.get("freqai", {}).get("follow_mode", False) + + if not follow_mode: + pair_dict = self.freqai.dd.pair_dict + else: + pair_dict = self.freqai.dd.follower_dict + + entry_tag = trade.enter_tag + + if ( + "prediction" + entry_tag not in pair_dict[pair] + or pair_dict[pair]['extras']["prediction" + entry_tag] == 0 + ): + pair_dict[pair]['extras']["prediction" + entry_tag] = abs(trade_candle["&-s_close"]) + if not follow_mode: + self.freqai.dd.save_drawer_to_disk() + else: + self.freqai.dd.save_follower_dict_to_disk() + + roi_price = pair_dict[pair]['extras']["prediction" + entry_tag] + roi_time = self.max_roi_time_long.value + + roi_decay = roi_price * ( + 1 - ((current_time - trade.open_date_utc).seconds) / (roi_time * 60) + ) + if roi_decay < 0: + roi_decay = self.linear_roi_offset.value + else: + roi_decay += self.linear_roi_offset.value + + if current_profit > roi_decay: + return "roi_custom_win" + + if current_profit < -roi_decay: + return "roi_custom_loss" + + def confirm_trade_exit( + self, + pair: str, + trade: Trade, + order_type: str, + amount: float, + rate: float, + time_in_force: str, + exit_reason: str, + current_time, + **kwargs, + ) -> bool: + + entry_tag = trade.enter_tag + follow_mode = self.config.get("freqai", {}).get("follow_mode", False) + if not follow_mode: + pair_dict = self.freqai.dd.pair_dict + else: + pair_dict = self.freqai.dd.follower_dict + + pair_dict[pair]['extras']["prediction" + entry_tag] = 0 + if not follow_mode: + self.freqai.dd.save_drawer_to_disk() + else: + self.freqai.dd.save_follower_dict_to_disk() + + return True + + def confirm_trade_entry( + self, + pair: str, + order_type: str, + amount: float, + rate: float, + time_in_force: str, + current_time, + entry_tag, + side: str, + **kwargs, + ) -> bool: + + df, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) + last_candle = df.iloc[-1].squeeze() + + if side == "long": + if rate > (last_candle["close"] * (1 + 0.0025)): + return False + else: + if rate < (last_candle["close"] * (1 - 0.0025)): + return False + + return True diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index 914aa964b..681af84c6 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -12,6 +12,7 @@ "tradable_balance_ratio": 0.99, "fiat_display_currency": "{{ fiat_display_currency }}",{{ ('\n "timeframe": "' + timeframe + '",') if timeframe else '' }} "dry_run": {{ dry_run | lower }}, + "dry_run_wallet": 1000, "cancel_open_orders_on_exit": false, "trading_mode": "{{ trading_mode }}", "margin_mode": "{{ margin_mode }}", diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index a7430c225..77444a023 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -30,7 +30,7 @@ "\n", "# Initialize empty configuration object\n", "config = Configuration.from_files([])\n", - "# Optionally, use existing configuration file\n", + "# Optionally (recommended), use existing configuration file\n", "# config = Configuration.from_files([\"config.json\"])\n", "\n", "# Define some constants\n", @@ -38,7 +38,7 @@ "# Name of the strategy class\n", "config[\"strategy\"] = \"SampleStrategy\"\n", "# Location of the data\n", - "data_location = Path(config['user_data_dir'], 'data', 'binance')\n", + "data_location = config['datadir']\n", "# Pair to analyze - Only use one pair here\n", "pair = \"BTC/USDT\"" ] @@ -365,7 +365,7 @@ "metadata": { "file_extension": ".py", "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.9.7 64-bit ('trade_397')", "language": "python", "name": "python3" }, @@ -379,7 +379,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.9.7" }, "mimetype": "text/x-python", "name": "python", @@ -427,7 +427,12 @@ ], "window_display": false }, - "version": 3 + "version": 3, + "vscode": { + "interpreter": { + "hash": "675f32a300d6d26767470181ad0b11dd4676bcce7ed1dd2ffe2fbc370c95fc7c" + } + } }, "nbformat": 4, "nbformat_minor": 4 diff --git a/freqtrade/templates/subtemplates/exchange_gateio.j2 b/freqtrade/templates/subtemplates/exchange_gateio.j2 new file mode 100644 index 000000000..63dff052f --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_gateio.j2 @@ -0,0 +1,12 @@ +"exchange": { + "name": "{{ exchange_name | lower }}", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "unknown_fee_rate": 1, + "ccxt_config": {}, + "ccxt_async_config": {}, + "pair_whitelist": [ + ], + "pair_blacklist": [ + ] +} diff --git a/freqtrade/templates/subtemplates/exchange_generic.j2 b/freqtrade/templates/subtemplates/exchange_generic.j2 index 08b11f365..01f637638 100644 --- a/freqtrade/templates/subtemplates/exchange_generic.j2 +++ b/freqtrade/templates/subtemplates/exchange_generic.j2 @@ -5,9 +5,7 @@ "ccxt_config": {}, "ccxt_async_config": {}, "pair_whitelist": [ - ], "pair_blacklist": [ - ] } diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 989f1d37a..488ca2fd7 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -247,12 +247,16 @@ def check_exit_timeout(self, pair: str, trade: 'Trade', order: 'Order', """ return False -def adjust_trade_position(self, trade: 'Trade', current_time: 'datetime', - current_rate: float, current_profit: float, min_stake: Optional[float], - max_stake: float, **kwargs) -> 'Optional[float]': +def adjust_trade_position(self, trade: 'Trade', current_time: datetime, + current_rate: float, current_profit: float, + min_stake: Optional[float], max_stake: float, + current_entry_rate: float, current_exit_rate: float, + current_entry_profit: float, current_exit_profit: float, + **kwargs) -> Optional[float]: """ - Custom trade adjustment logic, returning the stake amount that a trade should be increased. - This means extra buy orders with additional fees. + Custom trade adjustment logic, returning the stake amount that a trade should be + increased or decreased. + This means extra buy or sell orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -263,10 +267,16 @@ def adjust_trade_position(self, trade: 'Trade', current_time: 'datetime', :param current_time: datetime object, containing the current datetime :param current_rate: Current buy rate. :param current_profit: Current profit (as ratio), calculated based on current_rate. - :param min_stake: Minimal stake size allowed by exchange. - :param max_stake: Balance available for trading. + :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) + :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). + :param current_entry_rate: Current rate using entry pricing. + :param current_exit_rate: Current rate using exit pricing. + :param current_entry_profit: Current profit using entry pricing. + :param current_exit_profit: Current profit using exit pricing. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: Stake amount to adjust your trade + :return float: Stake amount to adjust your trade, + Positive values to increase position, Negative values to decrease position. + Return None for no action. """ return None diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py new file mode 100644 index 000000000..7980b7ca2 --- /dev/null +++ b/freqtrade/util/__init__.py @@ -0,0 +1,3 @@ +# flake8: noqa: F401 +from freqtrade.util.ft_precise import FtPrecise +from freqtrade.util.periodic_cache import PeriodicCache diff --git a/freqtrade/util/ft_precise.py b/freqtrade/util/ft_precise.py new file mode 100644 index 000000000..aba0517a9 --- /dev/null +++ b/freqtrade/util/ft_precise.py @@ -0,0 +1,12 @@ +""" +Slim wrapper around ccxt's Precise (string math) +To have imports from freqtrade - and support float initializers +""" +from ccxt import Precise + + +class FtPrecise(Precise): + def __init__(self, number, decimals=None): + if not isinstance(number, str): + number = str(number) + super().__init__(number, decimals) diff --git a/freqtrade/configuration/PeriodicCache.py b/freqtrade/util/periodic_cache.py similarity index 100% rename from freqtrade/configuration/PeriodicCache.py rename to freqtrade/util/periodic_cache.py diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 14e5a6743..41115c72e 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -148,7 +148,7 @@ class Wallets: # Position is not open ... continue size = self._exchange._contracts_to_amount(symbol, position['contracts']) - collateral = position['collateral'] + collateral = position['collateral'] or 0.0 leverage = position['leverage'] self._positions[symbol] = PositionWallet( symbol, position=size, diff --git a/mkdocs.yml b/mkdocs.yml index a43322f78..2e5e0c8c9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,7 @@ site_name: Freqtrade site_url: https://www.freqtrade.io/ repo_url: https://github.com/freqtrade/freqtrade +edit_uri: edit/develop/docs/ use_directory_urls: True nav: - Home: index.md @@ -32,9 +33,10 @@ nav: - Backtest analysis: advanced-backtesting.md - Advanced Topics: - Advanced Post-installation Tasks: advanced-setup.md - - Edge Positioning: edge.md - Advanced Strategy: strategy-advanced.md - Advanced Hyperopt: advanced-hyperopt.md + - FreqAI: freqai.md + - Edge Positioning: edge.md - Sandbox Testing: sandbox-testing.md - FAQ: faq.md - SQL Cheat-sheet: sql_cheatsheet.md diff --git a/requirements-dev.txt b/requirements-dev.txt index 3b98e20db..9c45e7277 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,10 +2,11 @@ -r requirements.txt -r requirements-plot.txt -r requirements-hyperopt.txt +-r requirements-freqai.txt -r docs/requirements-docs.txt coveralls==3.3.1 -flake8==4.0.1 +flake8==5.0.4 flake8-tidy-imports==4.8.0 mypy==0.971 pre-commit==2.20.0 @@ -16,14 +17,14 @@ pytest-mock==3.8.2 pytest-random-order==1.0.4 isort==5.10.1 # For datetime mocking -time-machine==2.7.1 +time-machine==2.8.1 # Convert jupyter notebooks to markdown documents -nbconvert==6.5.0 +nbconvert==6.5.3 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.3 +types-requests==2.28.9 types-tabulate==0.8.11 types-python-dateutil==2.8.19 diff --git a/requirements-freqai.txt b/requirements-freqai.txt new file mode 100644 index 000000000..26e4617af --- /dev/null +++ b/requirements-freqai.txt @@ -0,0 +1,8 @@ +# Include all requirements to run the bot. +-r requirements.txt + +# Required for freqai +scikit-learn==1.1.2 +joblib==1.1.0 +catboost==1.0.6; platform_machine != 'aarch64' +lightgbm==3.3.2 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 94e59ec15..020ccdda8 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,8 +2,8 @@ -r requirements.txt # Required for hyperopt -scipy==1.8.1 -scikit-learn==1.1.1 +scipy==1.9.0 +scikit-learn==1.1.2 scikit-optimize==0.9.0 -filelock==3.7.1 +filelock==3.8.0 progressbar2==4.0.0 diff --git a/requirements-plot.txt b/requirements-plot.txt index 0f6ae94c2..80cd3f4f2 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.9.0 +plotly==5.10.0 diff --git a/requirements.txt b/requirements.txt index b9e87749d..4a0531ea8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,18 @@ -numpy==1.23.1 +numpy==1.23.2 pandas==1.4.3 pandas-ta==0.3.14b -ccxt==1.91.29 +ccxt==1.92.52 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 -SQLAlchemy==1.4.39 +SQLAlchemy==1.4.40 python-telegram-bot==13.13 arrow==1.2.2 cachetools==4.2.2 requests==2.28.1 -urllib3==1.26.10 -jsonschema==4.7.2 +urllib3==1.26.11 +jsonschema==4.14.0 TA-Lib==0.4.24 technical==1.3.0 tabulate==0.8.10 @@ -28,13 +28,13 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.8 # Properly format api responses -orjson==3.7.8 +orjson==3.7.12 # Notify systemd sdnotify==0.3.2 # API Server -fastapi==0.79.0 +fastapi==0.79.1 uvicorn==0.18.2 pyjwt==2.4.0 aiofiles==0.8.0 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index e5d358c98..ac6d97133 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -275,14 +275,20 @@ class FtRestClient(): } return self._post("forceenter", data=data) - def forceexit(self, tradeid): + def forceexit(self, tradeid, ordertype=None, amount=None): """Force-exit a trade. :param tradeid: Id of the trade (can be received via status command) + :param ordertype: Order type to use (must be market or limit) + :param amount: Amount to sell. Full sell if not given :return: json object """ - return self._post("forceexit", data={"tradeid": tradeid}) + return self._post("forceexit", data={ + "tradeid": tradeid, + "ordertype": ordertype, + "amount": amount, + }) def strategies(self): """Lists available strategies @@ -355,6 +361,13 @@ class FtRestClient(): """ return self._get("sysinfo") + def health(self): + """Provides a quick health check of the running bot. + + :return: json object + """ + return self._get("health") + def add_arguments(): parser = argparse.ArgumentParser() diff --git a/setup.py b/setup.py index 7aa56bf81..8f04e75f7 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,13 @@ hyperopt = [ 'progressbar2', ] +freqai = [ + 'scikit-learn', + 'joblib', + 'catboost; platform_machine != "aarch64"', + 'lightgbm', +] + develop = [ 'coveralls', 'flake8', @@ -31,7 +38,7 @@ jupyter = [ 'nbconvert', ] -all_extra = plot + develop + jupyter + hyperopt +all_extra = plot + develop + jupyter + hyperopt + freqai setup( tests_require=[ @@ -42,7 +49,7 @@ setup( ], install_requires=[ # from requirements.txt - 'ccxt>=1.83.12', + 'ccxt>=1.92.9', 'SQLAlchemy', 'python-telegram-bot>=13.4', 'arrow>=0.17.0', @@ -79,6 +86,7 @@ setup( 'plot': plot, 'jupyter': jupyter, 'hyperopt': hyperopt, + 'freqai': freqai, 'all': all_extra, }, ) diff --git a/setup.sh b/setup.sh index 202cb70c7..1a4a285a3 100755 --- a/setup.sh +++ b/setup.sh @@ -77,7 +77,15 @@ function updateenv() { fi fi - ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} + REQUIREMENTS_FREQAI="" + read -p "Do you want to install dependencies for freqai [y/N]? " + dev=$REPLY + if [[ $REPLY =~ ^[Yy]$ ]] + then + REQUIREMENTS_FREQAI="-r requirements-freqai.txt" + fi + + ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} ${REQUIREMENTS_FREQAI} if [ $? -ne 0 ]; then echo "Failed installing dependencies" exit 1 diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index d6e80675e..28515a28a 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -638,7 +638,7 @@ def test_get_ui_download_url_direct(mocker): x, last_version = get_ui_download_url('0.0.3') -def test_download_data_keyboardInterrupt(mocker, caplog, markets): +def test_download_data_keyboardInterrupt(mocker, markets): dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) @@ -651,12 +651,15 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): "--pairs", "ETH/BTC", "XRP/BTC", ] with pytest.raises(SystemExit): - start_download_data(get_args(args)) + pargs = get_args(args) + pargs['config'] = None + + start_download_data(pargs) assert dl_mock.call_count == 1 -def test_download_data_timerange(mocker, caplog, markets): +def test_download_data_timerange(mocker, markets): dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) @@ -672,7 +675,9 @@ def test_download_data_timerange(mocker, caplog, markets): ] with pytest.raises(OperationalException, match=r"--days and --timerange are mutually.*"): - start_download_data(get_args(args)) + pargs = get_args(args) + pargs['config'] = None + start_download_data(pargs) assert dl_mock.call_count == 0 args = [ @@ -681,7 +686,9 @@ def test_download_data_timerange(mocker, caplog, markets): "--pairs", "ETH/BTC", "XRP/BTC", "--days", "20", ] - start_download_data(get_args(args)) + pargs = get_args(args) + pargs['config'] = None + start_download_data(pargs) assert dl_mock.call_count == 1 # 20days ago days_ago = arrow.get(arrow.now().shift(days=-20).date()).int_timestamp @@ -694,7 +701,9 @@ def test_download_data_timerange(mocker, caplog, markets): "--pairs", "ETH/BTC", "XRP/BTC", "--timerange", "20200101-" ] - start_download_data(get_args(args)) + pargs = get_args(args) + pargs['config'] = None + start_download_data(pargs) assert dl_mock.call_count == 1 assert dl_mock.call_args_list[0][1]['timerange'].startts == arrow.Arrow( @@ -1421,6 +1430,27 @@ def test_start_list_data(testdatadir, capsys): assert "\n| XRP/USDT | 1h | futures |\n" in captured.out assert "\n| XRP/USDT | 1h, 8h | mark |\n" in captured.out + args = [ + "list-data", + "--data-format-ohlcv", + "json", + "--pairs", "XRP/ETH", + "--datadir", + str(testdatadir), + "--show-timerange", + ] + pargs = get_args(args) + pargs['config'] = None + start_list_data(pargs) + captured = capsys.readouterr() + assert "Found 2 pair / timeframe combinations." in captured.out + assert ("\n| Pair | Timeframe | Type | From | To |\n" + in captured.out) + assert "UNITTEST/BTC" not in captured.out + assert ( + "\n| XRP/ETH | 1m | spot | 2019-10-11 00:00:00 | 2019-10-13 11:19:00 |\n" + in captured.out) + @pytest.mark.usefixtures("init_persistence") def test_show_trades(mocker, fee, capsys, caplog): diff --git a/tests/conftest.py b/tests/conftest.py index ff3e1007f..fffac8e0a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1627,8 +1627,8 @@ def limit_buy_order_open(): 'timestamp': arrow.utcnow().int_timestamp * 1000, 'datetime': arrow.utcnow().isoformat(), 'price': 0.00001099, + 'average': 0.00001099, 'amount': 90.99181073, - 'average': None, 'filled': 0.0, 'cost': 0.0009999, 'remaining': 90.99181073, @@ -2817,6 +2817,7 @@ def limit_buy_order_usdt_open(): 'datetime': arrow.utcnow().isoformat(), 'timestamp': arrow.utcnow().int_timestamp * 1000, 'price': 2.00, + 'average': 2.00, 'amount': 30.0, 'filled': 0.0, 'cost': 60.0, @@ -3084,416 +3085,416 @@ def leverage_tiers(): return { "1000SHIB/USDT": [ { - 'min': 0, - 'max': 50000, - 'mmr': 0.01, - 'lev': 50, + 'minNotional': 0, + 'maxNotional': 50000, + 'maintenanceMarginRate': 0.01, + 'maxLeverage': 50, 'maintAmt': 0.0 }, { - 'min': 50000, - 'max': 150000, - 'mmr': 0.025, - 'lev': 20, + 'minNotional': 50000, + 'maxNotional': 150000, + 'maintenanceMarginRate': 0.025, + 'maxLeverage': 20, 'maintAmt': 750.0 }, { - 'min': 150000, - 'max': 250000, - 'mmr': 0.05, - 'lev': 10, + 'minNotional': 150000, + 'maxNotional': 250000, + 'maintenanceMarginRate': 0.05, + 'maxLeverage': 10, 'maintAmt': 4500.0 }, { - 'min': 250000, - 'max': 500000, - 'mmr': 0.1, - 'lev': 5, + 'minNotional': 250000, + 'maxNotional': 500000, + 'maintenanceMarginRate': 0.1, + 'maxLeverage': 5, 'maintAmt': 17000.0 }, { - 'min': 500000, - 'max': 1000000, - 'mmr': 0.125, - 'lev': 4, + 'minNotional': 500000, + 'maxNotional': 1000000, + 'maintenanceMarginRate': 0.125, + 'maxLeverage': 4, 'maintAmt': 29500.0 }, { - 'min': 1000000, - 'max': 2000000, - 'mmr': 0.25, - 'lev': 2, + 'minNotional': 1000000, + 'maxNotional': 2000000, + 'maintenanceMarginRate': 0.25, + 'maxLeverage': 2, 'maintAmt': 154500.0 }, { - 'min': 2000000, - 'max': 30000000, - 'mmr': 0.5, - 'lev': 1, + 'minNotional': 2000000, + 'maxNotional': 30000000, + 'maintenanceMarginRate': 0.5, + 'maxLeverage': 1, 'maintAmt': 654500.0 }, ], "1INCH/USDT": [ { - 'min': 0, - 'max': 5000, - 'mmr': 0.012, - 'lev': 50, + 'minNotional': 0, + 'maxNotional': 5000, + 'maintenanceMarginRate': 0.012, + 'maxLeverage': 50, 'maintAmt': 0.0 }, { - 'min': 5000, - 'max': 25000, - 'mmr': 0.025, - 'lev': 20, + 'minNotional': 5000, + 'maxNotional': 25000, + 'maintenanceMarginRate': 0.025, + 'maxLeverage': 20, 'maintAmt': 65.0 }, { - 'min': 25000, - 'max': 100000, - 'mmr': 0.05, - 'lev': 10, + 'minNotional': 25000, + 'maxNotional': 100000, + 'maintenanceMarginRate': 0.05, + 'maxLeverage': 10, 'maintAmt': 690.0 }, { - 'min': 100000, - 'max': 250000, - 'mmr': 0.1, - 'lev': 5, + 'minNotional': 100000, + 'maxNotional': 250000, + 'maintenanceMarginRate': 0.1, + 'maxLeverage': 5, 'maintAmt': 5690.0 }, { - 'min': 250000, - 'max': 1000000, - 'mmr': 0.125, - 'lev': 2, + 'minNotional': 250000, + 'maxNotional': 1000000, + 'maintenanceMarginRate': 0.125, + 'maxLeverage': 2, 'maintAmt': 11940.0 }, { - 'min': 1000000, - 'max': 100000000, - 'mmr': 0.5, - 'lev': 1, + 'minNotional': 1000000, + 'maxNotional': 100000000, + 'maintenanceMarginRate': 0.5, + 'maxLeverage': 1, 'maintAmt': 386940.0 }, ], "AAVE/USDT": [ { - 'min': 0, - 'max': 5000, - 'mmr': 0.01, - 'lev': 50, + 'minNotional': 0, + 'maxNotional': 5000, + 'maintenanceMarginRate': 0.01, + 'maxLeverage': 50, 'maintAmt': 0.0 }, { - 'min': 5000, - 'max': 25000, - 'mmr': 0.02, - 'lev': 25, + 'minNotional': 5000, + 'maxNotional': 25000, + 'maintenanceMarginRate': 0.02, + 'maxLeverage': 25, 'maintAmt': 75.0 }, { - 'min': 25000, - 'max': 100000, - 'mmr': 0.05, - 'lev': 10, + 'minNotional': 25000, + 'maxNotional': 100000, + 'maintenanceMarginRate': 0.05, + 'maxLeverage': 10, 'maintAmt': 700.0 }, { - 'min': 100000, - 'max': 250000, - 'mmr': 0.1, - 'lev': 5, + 'minNotional': 100000, + 'maxNotional': 250000, + 'maintenanceMarginRate': 0.1, + 'maxLeverage': 5, 'maintAmt': 5700.0 }, { - 'min': 250000, - 'max': 1000000, - 'mmr': 0.125, - 'lev': 2, + 'minNotional': 250000, + 'maxNotional': 1000000, + 'maintenanceMarginRate': 0.125, + 'maxLeverage': 2, 'maintAmt': 11950.0 }, { - 'min': 10000000, - 'max': 50000000, - 'mmr': 0.5, - 'lev': 1, + 'minNotional': 10000000, + 'maxNotional': 50000000, + 'maintenanceMarginRate': 0.5, + 'maxLeverage': 1, 'maintAmt': 386950.0 }, ], "ADA/BUSD": [ { - "min": 0, - "max": 100000, - "mmr": 0.025, - "lev": 20, + "minNotional": 0, + "maxNotional": 100000, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20, "maintAmt": 0.0 }, { - "min": 100000, - "max": 500000, - "mmr": 0.05, - "lev": 10, + "minNotional": 100000, + "maxNotional": 500000, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10, "maintAmt": 2500.0 }, { - "min": 500000, - "max": 1000000, - "mmr": 0.1, - "lev": 5, + "minNotional": 500000, + "maxNotional": 1000000, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5, "maintAmt": 27500.0 }, { - "min": 1000000, - "max": 2000000, - "mmr": 0.15, - "lev": 3, + "minNotional": 1000000, + "maxNotional": 2000000, + "maintenanceMarginRate": 0.15, + "maxLeverage": 3, "maintAmt": 77500.0 }, { - "min": 2000000, - "max": 5000000, - "mmr": 0.25, - "lev": 2, + "minNotional": 2000000, + "maxNotional": 5000000, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2, "maintAmt": 277500.0 }, { - "min": 5000000, - "max": 30000000, - "mmr": 0.5, - "lev": 1, + "minNotional": 5000000, + "maxNotional": 30000000, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1, "maintAmt": 1527500.0 }, ], 'BNB/BUSD': [ { - "min": 0, # stake(before leverage) = 0 - "max": 100000, # max stake(before leverage) = 5000 - "mmr": 0.025, - "lev": 20, + "minNotional": 0, # stake(before leverage) = 0 + "maxNotional": 100000, # max stake(before leverage) = 5000 + "maintenanceMarginRate": 0.025, + "maxLeverage": 20, "maintAmt": 0.0 }, { - "min": 100000, # stake = 10000.0 - "max": 500000, # max_stake = 50000.0 - "mmr": 0.05, - "lev": 10, + "minNotional": 100000, # stake = 10000.0 + "maxNotional": 500000, # max_stake = 50000.0 + "maintenanceMarginRate": 0.05, + "maxLeverage": 10, "maintAmt": 2500.0 }, { - "min": 500000, # stake = 100000.0 - "max": 1000000, # max_stake = 200000.0 - "mmr": 0.1, - "lev": 5, + "minNotional": 500000, # stake = 100000.0 + "maxNotional": 1000000, # max_stake = 200000.0 + "maintenanceMarginRate": 0.1, + "maxLeverage": 5, "maintAmt": 27500.0 }, { - "min": 1000000, # stake = 333333.3333333333 - "max": 2000000, # max_stake = 666666.6666666666 - "mmr": 0.15, - "lev": 3, + "minNotional": 1000000, # stake = 333333.3333333333 + "maxNotional": 2000000, # max_stake = 666666.6666666666 + "maintenanceMarginRate": 0.15, + "maxLeverage": 3, "maintAmt": 77500.0 }, { - "min": 2000000, # stake = 1000000.0 - "max": 5000000, # max_stake = 2500000.0 - "mmr": 0.25, - "lev": 2, + "minNotional": 2000000, # stake = 1000000.0 + "maxNotional": 5000000, # max_stake = 2500000.0 + "maintenanceMarginRate": 0.25, + "maxLeverage": 2, "maintAmt": 277500.0 }, { - "min": 5000000, # stake = 5000000.0 - "max": 30000000, # max_stake = 30000000.0 - "mmr": 0.5, - "lev": 1, + "minNotional": 5000000, # stake = 5000000.0 + "maxNotional": 30000000, # max_stake = 30000000.0 + "maintenanceMarginRate": 0.5, + "maxLeverage": 1, "maintAmt": 1527500.0 } ], 'BNB/USDT': [ { - "min": 0, # stake = 0.0 - "max": 10000, # max_stake = 133.33333333333334 - "mmr": 0.0065, - "lev": 75, + "minNotional": 0, # stake = 0.0 + "maxNotional": 10000, # max_stake = 133.33333333333334 + "maintenanceMarginRate": 0.0065, + "maxLeverage": 75, "maintAmt": 0.0 }, { - "min": 10000, # stake = 200.0 - "max": 50000, # max_stake = 1000.0 - "mmr": 0.01, - "lev": 50, + "minNotional": 10000, # stake = 200.0 + "maxNotional": 50000, # max_stake = 1000.0 + "maintenanceMarginRate": 0.01, + "maxLeverage": 50, "maintAmt": 35.0 }, { - "min": 50000, # stake = 2000.0 - "max": 250000, # max_stake = 10000.0 - "mmr": 0.02, - "lev": 25, + "minNotional": 50000, # stake = 2000.0 + "maxNotional": 250000, # max_stake = 10000.0 + "maintenanceMarginRate": 0.02, + "maxLeverage": 25, "maintAmt": 535.0 }, { - "min": 250000, # stake = 25000.0 - "max": 1000000, # max_stake = 100000.0 - "mmr": 0.05, - "lev": 10, + "minNotional": 250000, # stake = 25000.0 + "maxNotional": 1000000, # max_stake = 100000.0 + "maintenanceMarginRate": 0.05, + "maxLeverage": 10, "maintAmt": 8035.0 }, { - "min": 1000000, # stake = 200000.0 - "max": 2000000, # max_stake = 400000.0 - "mmr": 0.1, - "lev": 5, + "minNotional": 1000000, # stake = 200000.0 + "maxNotional": 2000000, # max_stake = 400000.0 + "maintenanceMarginRate": 0.1, + "maxLeverage": 5, "maintAmt": 58035.0 }, { - "min": 2000000, # stake = 500000.0 - "max": 5000000, # max_stake = 1250000.0 - "mmr": 0.125, - "lev": 4, + "minNotional": 2000000, # stake = 500000.0 + "maxNotional": 5000000, # max_stake = 1250000.0 + "maintenanceMarginRate": 0.125, + "maxLeverage": 4, "maintAmt": 108035.0 }, { - "min": 5000000, # stake = 1666666.6666666667 - "max": 10000000, # max_stake = 3333333.3333333335 - "mmr": 0.15, - "lev": 3, + "minNotional": 5000000, # stake = 1666666.6666666667 + "maxNotional": 10000000, # max_stake = 3333333.3333333335 + "maintenanceMarginRate": 0.15, + "maxLeverage": 3, "maintAmt": 233035.0 }, { - "min": 10000000, # stake = 5000000.0 - "max": 20000000, # max_stake = 10000000.0 - "mmr": 0.25, - "lev": 2, + "minNotional": 10000000, # stake = 5000000.0 + "maxNotional": 20000000, # max_stake = 10000000.0 + "maintenanceMarginRate": 0.25, + "maxLeverage": 2, "maintAmt": 1233035.0 }, { - "min": 20000000, # stake = 20000000.0 - "max": 50000000, # max_stake = 50000000.0 - "mmr": 0.5, - "lev": 1, + "minNotional": 20000000, # stake = 20000000.0 + "maxNotional": 50000000, # max_stake = 50000000.0 + "maintenanceMarginRate": 0.5, + "maxLeverage": 1, "maintAmt": 6233035.0 }, ], 'BTC/USDT': [ { - "min": 0, # stake = 0.0 - "max": 50000, # max_stake = 400.0 - "mmr": 0.004, - "lev": 125, + "minNotional": 0, # stake = 0.0 + "maxNotional": 50000, # max_stake = 400.0 + "maintenanceMarginRate": 0.004, + "maxLeverage": 125, "maintAmt": 0.0 }, { - "min": 50000, # stake = 500.0 - "max": 250000, # max_stake = 2500.0 - "mmr": 0.005, - "lev": 100, + "minNotional": 50000, # stake = 500.0 + "maxNotional": 250000, # max_stake = 2500.0 + "maintenanceMarginRate": 0.005, + "maxLeverage": 100, "maintAmt": 50.0 }, { - "min": 250000, # stake = 5000.0 - "max": 1000000, # max_stake = 20000.0 - "mmr": 0.01, - "lev": 50, + "minNotional": 250000, # stake = 5000.0 + "maxNotional": 1000000, # max_stake = 20000.0 + "maintenanceMarginRate": 0.01, + "maxLeverage": 50, "maintAmt": 1300.0 }, { - "min": 1000000, # stake = 50000.0 - "max": 7500000, # max_stake = 375000.0 - "mmr": 0.025, - "lev": 20, + "minNotional": 1000000, # stake = 50000.0 + "maxNotional": 7500000, # max_stake = 375000.0 + "maintenanceMarginRate": 0.025, + "maxLeverage": 20, "maintAmt": 16300.0 }, { - "min": 7500000, # stake = 750000.0 - "max": 40000000, # max_stake = 4000000.0 - "mmr": 0.05, - "lev": 10, + "minNotional": 7500000, # stake = 750000.0 + "maxNotional": 40000000, # max_stake = 4000000.0 + "maintenanceMarginRate": 0.05, + "maxLeverage": 10, "maintAmt": 203800.0 }, { - "min": 40000000, # stake = 8000000.0 - "max": 100000000, # max_stake = 20000000.0 - "mmr": 0.1, - "lev": 5, + "minNotional": 40000000, # stake = 8000000.0 + "maxNotional": 100000000, # max_stake = 20000000.0 + "maintenanceMarginRate": 0.1, + "maxLeverage": 5, "maintAmt": 2203800.0 }, { - "min": 100000000, # stake = 25000000.0 - "max": 200000000, # max_stake = 50000000.0 - "mmr": 0.125, - "lev": 4, + "minNotional": 100000000, # stake = 25000000.0 + "maxNotional": 200000000, # max_stake = 50000000.0 + "maintenanceMarginRate": 0.125, + "maxLeverage": 4, "maintAmt": 4703800.0 }, { - "min": 200000000, # stake = 66666666.666666664 - "max": 400000000, # max_stake = 133333333.33333333 - "mmr": 0.15, - "lev": 3, + "minNotional": 200000000, # stake = 66666666.666666664 + "maxNotional": 400000000, # max_stake = 133333333.33333333 + "maintenanceMarginRate": 0.15, + "maxLeverage": 3, "maintAmt": 9703800.0 }, { - "min": 400000000, # stake = 200000000.0 - "max": 600000000, # max_stake = 300000000.0 - "mmr": 0.25, - "lev": 2, + "minNotional": 400000000, # stake = 200000000.0 + "maxNotional": 600000000, # max_stake = 300000000.0 + "maintenanceMarginRate": 0.25, + "maxLeverage": 2, "maintAmt": 4.97038E7 }, { - "min": 600000000, # stake = 600000000.0 - "max": 1000000000, # max_stake = 1000000000.0 - "mmr": 0.5, - "lev": 1, + "minNotional": 600000000, # stake = 600000000.0 + "maxNotional": 1000000000, # max_stake = 1000000000.0 + "maintenanceMarginRate": 0.5, + "maxLeverage": 1, "maintAmt": 1.997038E8 }, ], "ZEC/USDT": [ { - 'min': 0, - 'max': 50000, - 'mmr': 0.01, - 'lev': 50, + 'minNotional': 0, + 'maxNotional': 50000, + 'maintenanceMarginRate': 0.01, + 'maxLeverage': 50, 'maintAmt': 0.0 }, { - 'min': 50000, - 'max': 150000, - 'mmr': 0.025, - 'lev': 20, + 'minNotional': 50000, + 'maxNotional': 150000, + 'maintenanceMarginRate': 0.025, + 'maxLeverage': 20, 'maintAmt': 750.0 }, { - 'min': 150000, - 'max': 250000, - 'mmr': 0.05, - 'lev': 10, + 'minNotional': 150000, + 'maxNotional': 250000, + 'maintenanceMarginRate': 0.05, + 'maxLeverage': 10, 'maintAmt': 4500.0 }, { - 'min': 250000, - 'max': 500000, - 'mmr': 0.1, - 'lev': 5, + 'minNotional': 250000, + 'maxNotional': 500000, + 'maintenanceMarginRate': 0.1, + 'maxLeverage': 5, 'maintAmt': 17000.0 }, { - 'min': 500000, - 'max': 1000000, - 'mmr': 0.125, - 'lev': 4, + 'minNotional': 500000, + 'maxNotional': 1000000, + 'maintenanceMarginRate': 0.125, + 'maxLeverage': 4, 'maintAmt': 29500.0 }, { - 'min': 1000000, - 'max': 2000000, - 'mmr': 0.25, - 'lev': 2, + 'minNotional': 1000000, + 'maxNotional': 2000000, + 'maintenanceMarginRate': 0.25, + 'maxLeverage': 2, 'maintAmt': 154500.0 }, { - 'min': 2000000, - 'max': 30000000, - 'mmr': 0.5, - 'lev': 1, + 'minNotional': 2000000, + 'maxNotional': 30000000, + 'maintenanceMarginRate': 0.5, + 'maxLeverage': 1, 'maintAmt': 654500.0 }, ] diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index 1a8cf3183..9642435e5 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -214,7 +214,8 @@ def mock_trade_4(fee, is_short: bool): open_order_id=f'prod_buy_{direc(is_short)}_12345', strategy='StrategyTestV3', timeframe=5, - is_short=is_short + is_short=is_short, + stop_loss_pct=0.10 ) o = Order.parse_from_ccxt_object(mock_order_4(is_short), 'ETC/BTC', entry_side(is_short)) trade.orders.append(o) @@ -270,7 +271,8 @@ def mock_trade_5(fee, is_short: bool): enter_tag='TEST1', stoploss_order_id=f'prod_stoploss_{direc(is_short)}_3455', timeframe=5, - is_short=is_short + is_short=is_short, + stop_loss_pct=0.10, ) o = Order.parse_from_ccxt_object(mock_order_5(is_short), 'XRP/BTC', entry_side(is_short)) trade.orders.append(o) diff --git a/tests/conftest_trades_usdt.py b/tests/conftest_trades_usdt.py index 41d705c01..d54a416ef 100644 --- a/tests/conftest_trades_usdt.py +++ b/tests/conftest_trades_usdt.py @@ -63,7 +63,7 @@ def mock_trade_usdt_1(fee, is_short: bool): open_rate=10.0, close_rate=8.0, close_profit=-0.2, - close_profit_abs=-4.0, + close_profit_abs=-4.09, exchange='binance', strategy='SampleStrategy', open_order_id=f'prod_exit_1_{direc(is_short)}', @@ -81,7 +81,7 @@ def mock_trade_usdt_1(fee, is_short: bool): def mock_order_usdt_2(is_short: bool): return { 'id': f'1235_{direc(is_short)}', - 'symbol': 'ETC/USDT', + 'symbol': 'NEO/USDT', 'status': 'closed', 'side': entry_side(is_short), 'type': 'limit', @@ -95,7 +95,7 @@ def mock_order_usdt_2(is_short: bool): def mock_order_usdt_2_exit(is_short: bool): return { 'id': f'12366_{direc(is_short)}', - 'symbol': 'ETC/USDT', + 'symbol': 'NEO/USDT', 'status': 'closed', 'side': exit_side(is_short), 'type': 'limit', @@ -111,7 +111,7 @@ def mock_trade_usdt_2(fee, is_short: bool): Closed trade... """ trade = Trade( - pair='ETC/USDT', + pair='NEO/USDT', stake_amount=200.0, amount=100.0, amount_requested=100.0, @@ -132,10 +132,10 @@ def mock_trade_usdt_2(fee, is_short: bool): close_date=datetime.now(tz=timezone.utc) - timedelta(minutes=2), is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_2(is_short), 'ETC/USDT', entry_side(is_short)) + o = Order.parse_from_ccxt_object(mock_order_usdt_2(is_short), 'NEO/USDT', entry_side(is_short)) trade.orders.append(o) o = Order.parse_from_ccxt_object( - mock_order_usdt_2_exit(is_short), 'ETC/USDT', exit_side(is_short)) + mock_order_usdt_2_exit(is_short), 'NEO/USDT', exit_side(is_short)) trade.orders.append(o) return trade @@ -183,7 +183,7 @@ def mock_trade_usdt_3(fee, is_short: bool): open_rate=1.0, close_rate=1.1, close_profit=0.1, - close_profit_abs=9.8425, + close_profit_abs=2.8425, exchange='binance', is_open=False, strategy='StrategyTestV2', @@ -205,7 +205,7 @@ def mock_trade_usdt_3(fee, is_short: bool): def mock_order_usdt_4(is_short: bool): return { 'id': f'prod_buy_12345_{direc(is_short)}', - 'symbol': 'ETC/USDT', + 'symbol': 'NEO/USDT', 'status': 'open', 'side': entry_side(is_short), 'type': 'limit', @@ -221,7 +221,7 @@ def mock_trade_usdt_4(fee, is_short: bool): Simulate prod entry """ trade = Trade( - pair='ETC/USDT', + pair='NEO/USDT', stake_amount=20.0, amount=10.0, amount_requested=10.01, @@ -236,7 +236,7 @@ def mock_trade_usdt_4(fee, is_short: bool): timeframe=5, is_short=is_short, ) - o = Order.parse_from_ccxt_object(mock_order_usdt_4(is_short), 'ETC/USDT', entry_side(is_short)) + o = Order.parse_from_ccxt_object(mock_order_usdt_4(is_short), 'NEO/USDT', entry_side(is_short)) trade.orders.append(o) return trade diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 93f82de5d..49603feac 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -311,3 +311,27 @@ def test_no_exchange_mode(default_conf): with pytest.raises(OperationalException, match=message): dp.available_pairs() + + +def test_dp_send_msg(default_conf): + + default_conf["runmode"] = RunMode.DRY_RUN + + default_conf["timeframe"] = '1h' + dp = DataProvider(default_conf, None) + msg = 'Test message' + dp.send_msg(msg) + + assert msg in dp._msg_queue + dp._msg_queue.pop() + assert msg not in dp._msg_queue + # Message is not resent due to caching + dp.send_msg(msg) + assert msg not in dp._msg_queue + dp.send_msg(msg, always_send=True) + assert msg in dp._msg_queue + + default_conf["runmode"] = RunMode.BACKTEST + dp = DataProvider(default_conf, None) + dp.send_msg(msg, always_send=True) + assert msg not in dp._msg_queue diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index b30d6f998..1b0191fda 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -136,7 +136,7 @@ def test_adjust(mocker, edge_conf): )) pairs = ['A/B', 'C/D', 'E/F', 'G/H'] - assert(edge.adjust(pairs) == ['E/F', 'C/D']) + assert (edge.adjust(pairs) == ['E/F', 'C/D']) def test_stoploss(mocker, edge_conf): diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 45f8a3817..4d1c40647 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -376,96 +376,96 @@ def test_fill_leverage_tiers_binance(default_conf, mocker): assert exchange._leverage_tiers == { 'ADA/BUSD': [ { - "min": 0, - "max": 100000, - "mmr": 0.025, - "lev": 20, + "minNotional": 0, + "maxNotional": 100000, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20, "maintAmt": 0.0 }, { - "min": 100000, - "max": 500000, - "mmr": 0.05, - "lev": 10, + "minNotional": 100000, + "maxNotional": 500000, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10, "maintAmt": 2500.0 }, { - "min": 500000, - "max": 1000000, - "mmr": 0.1, - "lev": 5, + "minNotional": 500000, + "maxNotional": 1000000, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5, "maintAmt": 27500.0 }, { - "min": 1000000, - "max": 2000000, - "mmr": 0.15, - "lev": 3, + "minNotional": 1000000, + "maxNotional": 2000000, + "maintenanceMarginRate": 0.15, + "maxLeverage": 3, "maintAmt": 77500.0 }, { - "min": 2000000, - "max": 5000000, - "mmr": 0.25, - "lev": 2, + "minNotional": 2000000, + "maxNotional": 5000000, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2, "maintAmt": 277500.0 }, { - "min": 5000000, - "max": 30000000, - "mmr": 0.5, - "lev": 1, + "minNotional": 5000000, + "maxNotional": 30000000, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1, "maintAmt": 1527500.0 } ], "ZEC/USDT": [ { - 'min': 0, - 'max': 50000, - 'mmr': 0.01, - 'lev': 50, + 'minNotional': 0, + 'maxNotional': 50000, + 'maintenanceMarginRate': 0.01, + 'maxLeverage': 50, 'maintAmt': 0.0 }, { - 'min': 50000, - 'max': 150000, - 'mmr': 0.025, - 'lev': 20, + 'minNotional': 50000, + 'maxNotional': 150000, + 'maintenanceMarginRate': 0.025, + 'maxLeverage': 20, 'maintAmt': 750.0 }, { - 'min': 150000, - 'max': 250000, - 'mmr': 0.05, - 'lev': 10, + 'minNotional': 150000, + 'maxNotional': 250000, + 'maintenanceMarginRate': 0.05, + 'maxLeverage': 10, 'maintAmt': 4500.0 }, { - 'min': 250000, - 'max': 500000, - 'mmr': 0.1, - 'lev': 5, + 'minNotional': 250000, + 'maxNotional': 500000, + 'maintenanceMarginRate': 0.1, + 'maxLeverage': 5, 'maintAmt': 17000.0 }, { - 'min': 500000, - 'max': 1000000, - 'mmr': 0.125, - 'lev': 4, + 'minNotional': 500000, + 'maxNotional': 1000000, + 'maintenanceMarginRate': 0.125, + 'maxLeverage': 4, 'maintAmt': 29500.0 }, { - 'min': 1000000, - 'max': 2000000, - 'mmr': 0.25, - 'lev': 2, + 'minNotional': 1000000, + 'maxNotional': 2000000, + 'maintenanceMarginRate': 0.25, + 'maxLeverage': 2, 'maintAmt': 154500.0 }, { - 'min': 2000000, - 'max': 30000000, - 'mmr': 0.5, - 'lev': 1, + 'minNotional': 2000000, + 'maxNotional': 30000000, + 'maintenanceMarginRate': 0.5, + 'maxLeverage': 1, 'maintAmt': 654500.0 }, ] diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 7bb52ccaf..49b7684f8 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -137,6 +137,10 @@ def exchange_futures(request, exchange_conf, class_mocker): 'freqtrade.exchange.binance.Binance.fill_leverage_tiers') class_mocker.patch('freqtrade.exchange.exchange.Exchange.fetch_trading_fees') class_mocker.patch('freqtrade.exchange.okx.Okx.additional_exchange_init') + class_mocker.patch('freqtrade.exchange.exchange.Exchange.load_cached_leverage_tiers', + return_value=None) + class_mocker.patch('freqtrade.exchange.exchange.Exchange.cache_leverage_tiers') + exchange = ExchangeResolver.load_exchange( request.param, exchange_conf, validate=True, load_leverage_tiers=True) @@ -405,14 +409,14 @@ class TestCCXTExchange(): assert (isinstance(futures_leverage, float) or isinstance(futures_leverage, int)) assert futures_leverage >= 1.0 - def test_ccxt__get_contract_size(self, exchange_futures): + def test_ccxt_get_contract_size(self, exchange_futures): futures, futures_name = exchange_futures if futures: futures_pair = EXCHANGES[futures_name].get( 'futures_pair', EXCHANGES[futures_name]['pair'] ) - contract_size = futures._get_contract_size(futures_pair) + contract_size = futures.get_contract_size(futures_pair) assert (isinstance(contract_size, float) or isinstance(contract_size, int)) assert contract_size >= 0.0 @@ -464,6 +468,7 @@ class TestCCXTExchange(): False, 100, 100, + 100, ) assert (isinstance(liquidation_price, float)) assert liquidation_price >= 0.0 @@ -474,6 +479,7 @@ class TestCCXTExchange(): False, 100, 100, + 100, ) assert (isinstance(liquidation_price, float)) assert liquidation_price >= 0.0 diff --git a/tests/exchange/test_ccxt_precise.py b/tests/exchange/test_ccxt_precise.py index 026adb4c1..5542ac8d2 100644 --- a/tests/exchange/test_ccxt_precise.py +++ b/tests/exchange/test_ccxt_precise.py @@ -1,14 +1,14 @@ -from ccxt import Precise +from freqtrade.util import FtPrecise -ws = Precise('-1.123e-6') -ws = Precise('-1.123e-6') -xs = Precise('0.00000002') -ys = Precise('69696900000') -zs = Precise('0') +ws = FtPrecise('-1.123e-6') +ws = FtPrecise('-1.123e-6') +xs = FtPrecise('0.00000002') +ys = FtPrecise('69696900000') +zs = FtPrecise('0') -def test_precise(): +def test_FtPrecise(): assert ys * xs == '1393.938' assert xs * ys == '1393.938' @@ -45,31 +45,38 @@ def test_precise(): assert xs + zs == '0.00000002' assert ys + zs == '69696900000' - assert abs(Precise('-500.1')) == '500.1' - assert abs(Precise('213')) == '213' + assert abs(FtPrecise('-500.1')) == '500.1' + assert abs(FtPrecise('213')) == '213' - assert abs(Precise('-500.1')) == '500.1' - assert -Precise('213') == '-213' + assert abs(FtPrecise('-500.1')) == '500.1' + assert -FtPrecise('213') == '-213' - assert Precise('10.1') % Precise('0.5') == '0.1' - assert Precise('5550') % Precise('120') == '30' + assert FtPrecise('10.1') % FtPrecise('0.5') == '0.1' + assert FtPrecise('5550') % FtPrecise('120') == '30' - assert Precise('-0.0') == Precise('0') - assert Precise('5.534000') == Precise('5.5340') + assert FtPrecise('-0.0') == FtPrecise('0') + assert FtPrecise('5.534000') == FtPrecise('5.5340') - assert min(Precise('-3.1415'), Precise('-2')) == '-3.1415' + assert min(FtPrecise('-3.1415'), FtPrecise('-2')) == '-3.1415' - assert max(Precise('3.1415'), Precise('-2')) == '3.1415' + assert max(FtPrecise('3.1415'), FtPrecise('-2')) == '3.1415' - assert Precise('2') > Precise('1.2345') - assert not Precise('-3.1415') > Precise('-2') - assert not Precise('3.1415') > Precise('3.1415') - assert Precise.string_gt('3.14150000000000000000001', '3.1415') + assert FtPrecise('2') > FtPrecise('1.2345') + assert not FtPrecise('-3.1415') > FtPrecise('-2') + assert not FtPrecise('3.1415') > FtPrecise('3.1415') + assert FtPrecise.string_gt('3.14150000000000000000001', '3.1415') - assert Precise('3.1415') >= Precise('3.1415') - assert Precise('3.14150000000000000000001') >= Precise('3.1415') + assert FtPrecise('3.1415') >= FtPrecise('3.1415') + assert FtPrecise('3.14150000000000000000001') >= FtPrecise('3.1415') - assert not Precise('3.1415') < Precise('3.1415') + assert not FtPrecise('3.1415') < FtPrecise('3.1415') - assert Precise('3.1415') <= Precise('3.1415') - assert Precise('3.1415') <= Precise('3.14150000000000000000001') + assert FtPrecise('3.1415') <= FtPrecise('3.1415') + assert FtPrecise('3.1415') <= FtPrecise('3.14150000000000000000001') + + assert FtPrecise(213) == '213' + assert FtPrecise(-213) == '-213' + assert str(FtPrecise(-213)) == '-213' + assert FtPrecise(213.2) == '213.2' + assert float(FtPrecise(213.2)) == 213.2 + assert float(FtPrecise(-213.2)) == -213.2 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 9252040ea..093284668 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -14,12 +14,12 @@ from pandas import DataFrame from freqtrade.enums import CandleType, MarginMode, TradingMode from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException, OperationalException, PricingError, TemporaryError) -from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken +from freqtrade.exchange import (Binance, Bittrex, Exchange, Kraken, amount_to_precision, + date_minus_candles, market_is_active, price_to_precision, + timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, + timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, calculate_backoff, remove_credentials) -from freqtrade.exchange.exchange import (date_minus_candles, market_is_active, timeframe_to_minutes, - timeframe_to_msecs, timeframe_to_next_date, - timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re @@ -27,6 +27,57 @@ from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has # Make sure to always keep one exchange here which is NOT subclassed!! EXCHANGES = ['bittrex', 'binance', 'kraken', 'ftx', 'gateio'] +get_entry_rate_data = [ + ('other', 20, 19, 10, 0.0, 20), # Full ask side + ('ask', 20, 19, 10, 0.0, 20), # Full ask side + ('ask', 20, 19, 10, 1.0, 10), # Full last side + ('ask', 20, 19, 10, 0.5, 15), # Between ask and last + ('ask', 20, 19, 10, 0.7, 13), # Between ask and last + ('ask', 20, 19, 10, 0.3, 17), # Between ask and last + ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask + ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask + ('ask', 20, 19, 10, None, 20), # price_last_balance missing + ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask + ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask + ('ask', 4, 5, None, 1, 4), # last not available - uses ask + ('ask', 4, 5, None, 0, 4), # last not available - uses ask + ('same', 21, 20, 10, 0.0, 20), # Full bid side + ('bid', 21, 20, 10, 0.0, 20), # Full bid side + ('bid', 21, 20, 10, 1.0, 10), # Full last side + ('bid', 21, 20, 10, 0.5, 15), # Between bid and last + ('bid', 21, 20, 10, 0.7, 13), # Between bid and last + ('bid', 21, 20, 10, 0.3, 17), # Between bid and last + ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid + ('bid', 21, 20, 10, None, 20), # price_last_balance missing + ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid + ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid + ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid + ('bid', 6, 5, None, 1, 5), # last not available - uses bid + ('bid', 6, 5, None, 0, 5), # last not available - uses bid +] + +get_sell_rate_data = [ + ('bid', 12.0, 11.0, 11.5, 0.0, 11.0), # full bid side + ('bid', 12.0, 11.0, 11.5, 1.0, 11.5), # full last side + ('bid', 12.0, 11.0, 11.5, 0.5, 11.25), # between bid and lat + ('bid', 12.0, 11.2, 10.5, 0.0, 11.2), # Last smaller than bid + ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid + ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid + ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), + ('bid', 0.003, 0.002, 0.005, None, 0.002), + ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side + ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side + ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat + ('ask', 12.2, 11.2, 10.5, 0.0, 12.2), # Last smaller than ask + ('ask', 12.0, 11.0, 10.5, 1.0, 12.0), # Last smaller than ask - uses ask + ('ask', 12.0, 11.2, 10.5, 0.5, 12.0), # Last smaller than ask - uses ask + ('ask', 10.0, 11.0, 11.0, 0.0, 10.0), + ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), + ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), + ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), + ('ask', 0.006, 1.0, 11.0, None, 0.006), +] + def ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, fun, mock_ccxt_fun, retries=API_RETRY_COUNT + 1, **kwargs): @@ -130,11 +181,11 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog): assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) assert log_has(asynclogmsg, caplog) # Test additional headers case - Exchange._headers = {'hello': 'world'} + Exchange._ccxt_params = {'hello': 'world'} ex = Exchange(conf) assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) - assert ex._api.headers == {'hello': 'world'} + assert ex._api.hello == 'world' assert ex._ccxt_config == {} Exchange._headers = {} @@ -228,62 +279,35 @@ def test_validate_order_time_in_force(default_conf, mocker, caplog): ex.validate_order_time_in_force(tif2) -@pytest.mark.parametrize("amount,precision_mode,precision,contract_size,expected,trading_mode", [ - (2.34559, 2, 4, 1, 2.3455, 'spot'), - (2.34559, 2, 5, 1, 2.34559, 'spot'), - (2.34559, 2, 3, 1, 2.345, 'spot'), - (2.9999, 2, 3, 1, 2.999, 'spot'), - (2.9909, 2, 3, 1, 2.990, 'spot'), - (2.9909, 2, 0, 1, 2, 'spot'), - (29991.5555, 2, 0, 1, 29991, 'spot'), - (29991.5555, 2, -1, 1, 29990, 'spot'), - (29991.5555, 2, -2, 1, 29900, 'spot'), +@pytest.mark.parametrize("amount,precision_mode,precision,expected", [ + (2.34559, 2, 4, 2.3455), + (2.34559, 2, 5, 2.34559), + (2.34559, 2, 3, 2.345), + (2.9999, 2, 3, 2.999), + (2.9909, 2, 3, 2.990), + (2.9909, 2, 0, 2), + (29991.5555, 2, 0, 29991), + (29991.5555, 2, -1, 29990), + (29991.5555, 2, -2, 29900), # Tests for Tick-size - (2.34559, 4, 0.0001, 1, 2.3455, 'spot'), - (2.34559, 4, 0.00001, 1, 2.34559, 'spot'), - (2.34559, 4, 0.001, 1, 2.345, 'spot'), - (2.9999, 4, 0.001, 1, 2.999, 'spot'), - (2.9909, 4, 0.001, 1, 2.990, 'spot'), - (2.9909, 4, 0.005, 0.01, 2.99, 'futures'), - (2.9999, 4, 0.005, 10, 2.995, 'futures'), + (2.34559, 4, 0.0001, 2.3455), + (2.34559, 4, 0.00001, 2.34559), + (2.34559, 4, 0.001, 2.345), + (2.9999, 4, 0.001, 2.999), + (2.9909, 4, 0.001, 2.990), + (2.9909, 4, 0.005, 2.99), + (2.9999, 4, 0.005, 2.995), ]) -def test_amount_to_precision( - default_conf, - mocker, - amount, - precision_mode, - precision, - contract_size, - expected, - trading_mode -): +def test_amount_to_precision(amount, precision_mode, precision, expected,): """ Test rounds down """ - - markets = PropertyMock(return_value={ - 'ETH/BTC': { - 'contractSize': contract_size, - 'precision': { - 'amount': precision - } - } - }) - - default_conf['trading_mode'] = trading_mode - default_conf['margin_mode'] = 'isolated' - - exchange = get_patched_exchange(mocker, default_conf, id="binance") # digits counting mode # DECIMAL_PLACES = 2 # SIGNIFICANT_DIGITS = 3 # TICK_SIZE = 4 - mocker.patch('freqtrade.exchange.Exchange.precisionMode', - PropertyMock(return_value=precision_mode)) - mocker.patch('freqtrade.exchange.Exchange.markets', markets) - pair = 'ETH/BTC' - assert exchange.amount_to_precision(pair, amount) == expected + assert amount_to_precision(amount, precision, precision_mode) == expected @pytest.mark.parametrize("price,precision_mode,precision,expected", [ @@ -308,21 +332,13 @@ def test_amount_to_precision( (0.000000003483, 4, 1e-12, 0.000000003483), ]) -def test_price_to_precision(default_conf, mocker, price, precision_mode, precision, expected): - """Test price to precision""" - markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'price': precision}}}) - - exchange = get_patched_exchange(mocker, default_conf, id="binance") - mocker.patch('freqtrade.exchange.Exchange.markets', markets) +def test_price_to_precision(price, precision_mode, precision, expected): # digits counting mode # DECIMAL_PLACES = 2 # SIGNIFICANT_DIGITS = 3 # TICK_SIZE = 4 - mocker.patch('freqtrade.exchange.Exchange.precisionMode', - PropertyMock(return_value=precision_mode)) - pair = 'ETH/BTC' - assert exchange.price_to_precision(pair, price) == expected + assert price_to_precision(price, precision, precision_mode) == expected @pytest.mark.parametrize("price,precision_mode,precision,expected", [ @@ -2336,10 +2352,11 @@ def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name) order_book = exchange.fetch_l2_order_book(pair='ETH/BTC', limit=val) assert api_mock.fetch_l2_order_book.call_args_list[0][0][0] == 'ETH/BTC' # Not all exchanges support all limits for orderbook - if not exchange._ft_has['l2_limit_range'] or val in exchange._ft_has['l2_limit_range']: + if (not exchange.get_option('l2_limit_range') + or val in exchange.get_option('l2_limit_range')): assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == val else: - next_limit = exchange.get_next_limit_in_list(val, exchange._ft_has['l2_limit_range']) + next_limit = exchange.get_next_limit_in_list(val, exchange.get_option('l2_limit_range')) assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == next_limit @@ -2360,34 +2377,7 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50) -@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [ - ('other', 20, 19, 10, 0.0, 20), # Full ask side - ('ask', 20, 19, 10, 0.0, 20), # Full ask side - ('ask', 20, 19, 10, 1.0, 10), # Full last side - ('ask', 20, 19, 10, 0.5, 15), # Between ask and last - ('ask', 20, 19, 10, 0.7, 13), # Between ask and last - ('ask', 20, 19, 10, 0.3, 17), # Between ask and last - ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask - ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask - ('ask', 20, 19, 10, None, 20), # price_last_balance missing - ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask - ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask - ('ask', 4, 5, None, 1, 4), # last not available - uses ask - ('ask', 4, 5, None, 0, 4), # last not available - uses ask - ('same', 21, 20, 10, 0.0, 20), # Full bid side - ('bid', 21, 20, 10, 0.0, 20), # Full bid side - ('bid', 21, 20, 10, 1.0, 10), # Full last side - ('bid', 21, 20, 10, 0.5, 15), # Between bid and last - ('bid', 21, 20, 10, 0.7, 13), # Between bid and last - ('bid', 21, 20, 10, 0.3, 17), # Between bid and last - ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid - ('bid', 21, 20, 10, None, 20), # price_last_balance missing - ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid - ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid - ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid - ('bid', 6, 5, None, 1, 5), # last not available - uses bid - ('bid', 6, 5, None, 0, 5), # last not available - uses bid -]) +@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", get_entry_rate_data) def test_get_entry_rate(mocker, default_conf, caplog, side, ask, bid, last, last_ab, expected) -> None: caplog.set_level(logging.DEBUG) @@ -2411,27 +2401,7 @@ def test_get_entry_rate(mocker, default_conf, caplog, side, ask, bid, assert not log_has("Using cached entry rate for ETH/BTC.", caplog) -@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', [ - ('bid', 12.0, 11.0, 11.5, 0.0, 11.0), # full bid side - ('bid', 12.0, 11.0, 11.5, 1.0, 11.5), # full last side - ('bid', 12.0, 11.0, 11.5, 0.5, 11.25), # between bid and lat - ('bid', 12.0, 11.2, 10.5, 0.0, 11.2), # Last smaller than bid - ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid - ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid - ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), - ('bid', 0.003, 0.002, 0.005, None, 0.002), - ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side - ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side - ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat - ('ask', 12.2, 11.2, 10.5, 0.0, 12.2), # Last smaller than ask - ('ask', 12.0, 11.0, 10.5, 1.0, 12.0), # Last smaller than ask - uses ask - ('ask', 12.0, 11.2, 10.5, 0.5, 12.0), # Last smaller than ask - uses ask - ('ask', 10.0, 11.0, 11.0, 0.0, 10.0), - ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), - ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), - ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), - ('ask', 0.006, 1.0, 11.0, None, 0.006), -]) +@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', get_sell_rate_data) def test_get_exit_rate(default_conf, mocker, caplog, side, bid, ask, last, last_ab, expected) -> None: caplog.set_level(logging.DEBUG) @@ -2481,14 +2451,14 @@ def test_get_ticker_rate_error(mocker, entry, default_conf, caplog, side, is_sho @pytest.mark.parametrize('is_short,side,expected', [ - (False, 'bid', 0.043936), # Value from order_book_l2 fitxure - bids side - (False, 'ask', 0.043949), # Value from order_book_l2 fitxure - asks side - (False, 'other', 0.043936), # Value from order_book_l2 fitxure - bids side - (False, 'same', 0.043949), # Value from order_book_l2 fitxure - asks side - (True, 'bid', 0.043936), # Value from order_book_l2 fitxure - bids side - (True, 'ask', 0.043949), # Value from order_book_l2 fitxure - asks side - (True, 'other', 0.043949), # Value from order_book_l2 fitxure - asks side - (True, 'same', 0.043936), # Value from order_book_l2 fitxure - bids side + (False, 'bid', 0.043936), # Value from order_book_l2 fixture - bids side + (False, 'ask', 0.043949), # Value from order_book_l2 fixture - asks side + (False, 'other', 0.043936), # Value from order_book_l2 fixture - bids side + (False, 'same', 0.043949), # Value from order_book_l2 fixture - asks side + (True, 'bid', 0.043936), # Value from order_book_l2 fixture - bids side + (True, 'ask', 0.043949), # Value from order_book_l2 fixture - asks side + (True, 'other', 0.043949), # Value from order_book_l2 fixture - asks side + (True, 'same', 0.043936), # Value from order_book_l2 fixture - bids side ]) def test_get_exit_rate_orderbook( default_conf, mocker, caplog, is_short, side, expected, order_book_l2): @@ -2521,7 +2491,8 @@ def test_get_exit_rate_orderbook_exception(default_conf, mocker, caplog): exchange = get_patched_exchange(mocker, default_conf) with pytest.raises(PricingError): exchange.get_rate(pair, refresh=True, side="exit", is_short=False) - assert log_has_re(r"Exit Price at location 1 from orderbook could not be determined\..*", + assert log_has_re(rf"{pair} - Exit Price at location 1 from orderbook " + rf"could not be determined\..*", caplog) @@ -2548,6 +2519,84 @@ def test_get_exit_rate_exception(default_conf, mocker, is_short): assert exchange.get_rate(pair, refresh=True, side="exit", is_short=is_short) == 0.13 +@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", get_entry_rate_data) +@pytest.mark.parametrize("side2", ['bid', 'ask']) +@pytest.mark.parametrize("use_order_book", [True, False]) +def test_get_rates_testing_buy(mocker, default_conf, caplog, side, ask, bid, + last, last_ab, expected, + side2, use_order_book, order_book_l2) -> None: + caplog.set_level(logging.DEBUG) + if last_ab is None: + del default_conf['entry_pricing']['price_last_balance'] + else: + default_conf['entry_pricing']['price_last_balance'] = last_ab + default_conf['entry_pricing']['price_side'] = side + default_conf['exit_pricing']['price_side'] = side2 + default_conf['exit_pricing']['use_order_book'] = use_order_book + api_mock = MagicMock() + api_mock.fetch_l2_order_book = order_book_l2 + api_mock.fetch_ticker = MagicMock( + return_value={'ask': ask, 'last': last, 'bid': bid}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + + assert exchange.get_rates('ETH/BTC', refresh=True, is_short=False)[0] == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + api_mock.fetch_l2_order_book.reset_mock() + api_mock.fetch_ticker.reset_mock() + assert exchange.get_rates('ETH/BTC', refresh=False, is_short=False)[0] == expected + assert log_has("Using cached buy rate for ETH/BTC.", caplog) + assert api_mock.fetch_l2_order_book.call_count == 0 + assert api_mock.fetch_ticker.call_count == 0 + # Running a 2nd time with Refresh on! + caplog.clear() + + assert exchange.get_rates('ETH/BTC', refresh=True, is_short=False)[0] == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + assert api_mock.fetch_l2_order_book.call_count == int(use_order_book) + assert api_mock.fetch_ticker.call_count == 1 + + +@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', get_sell_rate_data) +@pytest.mark.parametrize("side2", ['bid', 'ask']) +@pytest.mark.parametrize("use_order_book", [True, False]) +def test_get_rates_testing_sell(default_conf, mocker, caplog, side, bid, ask, + last, last_ab, expected, + side2, use_order_book, order_book_l2) -> None: + caplog.set_level(logging.DEBUG) + + default_conf['exit_pricing']['price_side'] = side + if last_ab is not None: + default_conf['exit_pricing']['price_last_balance'] = last_ab + + default_conf['entry_pricing']['price_side'] = side2 + default_conf['entry_pricing']['use_order_book'] = use_order_book + api_mock = MagicMock() + api_mock.fetch_l2_order_book = order_book_l2 + api_mock.fetch_ticker = MagicMock( + return_value={'ask': ask, 'last': last, 'bid': bid}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + + pair = "ETH/BTC" + + # Test regular mode + rate = exchange.get_rates(pair, refresh=True, is_short=False)[1] + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) + assert isinstance(rate, float) + assert rate == expected + # Use caching + api_mock.fetch_l2_order_book.reset_mock() + api_mock.fetch_ticker.reset_mock() + + rate = exchange.get_rates(pair, refresh=False, is_short=False)[1] + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + assert api_mock.fetch_l2_order_book.call_count == 0 + assert api_mock.fetch_ticker.call_count == 0 + + @pytest.mark.parametrize("exchange_name", EXCHANGES) @pytest.mark.asyncio async def test___async_get_candle_history_sort(default_conf, mocker, exchange_name): @@ -3263,16 +3312,16 @@ def test_merge_ft_has_dict(default_conf, mocker): ex = Kraken(default_conf) assert ex._ft_has != Exchange._ft_has_default - assert ex._ft_has['trades_pagination'] == 'id' - assert ex._ft_has['trades_pagination_arg'] == 'since' + assert ex.get_option('trades_pagination') == 'id' + assert ex.get_option('trades_pagination_arg') == 'since' # Binance defines different values ex = Binance(default_conf) assert ex._ft_has != Exchange._ft_has_default - assert ex._ft_has['stoploss_on_exchange'] - assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc'] - assert ex._ft_has['trades_pagination'] == 'id' - assert ex._ft_has['trades_pagination_arg'] == 'fromId' + assert ex.get_option('stoploss_on_exchange') + assert ex.get_option('order_time_in_force') == ['gtc', 'fok', 'ioc'] + assert ex.get_option('trades_pagination') == 'id' + assert ex.get_option('trades_pagination_arg') == 'fromId' conf = copy.deepcopy(default_conf) conf['exchange']['_ft_has_params'] = {"DeadBeef": 20, @@ -3727,8 +3776,8 @@ def test__get_funding_fees_from_exchange(default_conf, mocker, exchange_name): since=unix_time ) - assert(isclose(expected_fees, fees_from_datetime)) - assert(isclose(expected_fees, fees_from_unix_time)) + assert (isclose(expected_fees, fees_from_datetime)) + assert (isclose(expected_fees, fees_from_unix_time)) ccxt_exceptionhandlers( mocker, @@ -4083,7 +4132,8 @@ def test_get_or_calculate_liquidation_price(mocker, default_conf): pair='NEAR/USDT:USDT', open_rate=18.884, is_short=False, - position=0.8, + amount=0.8, + stake_amount=18.884 * 0.8, wallet_balance=0.8, ) assert liq_price == 17.47 @@ -4094,25 +4144,12 @@ def test_get_or_calculate_liquidation_price(mocker, default_conf): pair='NEAR/USDT:USDT', open_rate=18.884, is_short=False, - position=0.8, + amount=0.8, + stake_amount=18.884 * 0.8, wallet_balance=0.8, ) assert liq_price == 17.540699999999998 - ccxt_exceptionhandlers( - mocker, - default_conf, - api_mock, - "binance", - "get_or_calculate_liquidation_price", - "fetch_positions", - pair="XRP/USDT", - open_rate=0.0, - is_short=False, - position=0.0, - wallet_balance=0.0, - ) - @pytest.mark.parametrize('exchange,rate_start,rate_end,d1,d2,amount,expected_fees', [ ('binance', 0, 2, "2021-09-01 01:00:00", "2021-09-01 04:00:00", 30.0, 0.0), @@ -4253,7 +4290,7 @@ def test__fetch_and_calculate_funding_fees_datetime_called( ('XLTCUSDT', 0.01, 'futures'), ('ETH/USDT:USDT', 10, 'futures') ]) -def test__get_contract_size(mocker, default_conf, pair, expected_size, trading_mode): +def est__get_contract_size(mocker, default_conf, pair, expected_size, trading_mode): api_mock = MagicMock() default_conf['trading_mode'] = trading_mode default_conf['margin_mode'] = 'isolated' @@ -4272,7 +4309,7 @@ def test__get_contract_size(mocker, default_conf, pair, expected_size, trading_m 'contractSize': '10', } }) - size = exchange._get_contract_size(pair) + size = exchange.get_contract_size(pair) assert expected_size == size @@ -4508,7 +4545,8 @@ def test_liquidation_price_is_none( pair='DOGE/USDT', open_rate=open_rate, is_short=is_short, - position=71200.81144, + amount=71200.81144, + stake_amount=open_rate * 71200.81144, wallet_balance=-56354.57, mm_ex_1=0.10, upnl_ex_1=0.0 @@ -4517,7 +4555,7 @@ def test_liquidation_price_is_none( @pytest.mark.parametrize( 'exchange_name, is_short, trading_mode, margin_mode, wallet_balance, ' - 'mm_ex_1, upnl_ex_1, maintenance_amt, position, open_rate, ' + 'mm_ex_1, upnl_ex_1, maintenance_amt, amount, open_rate, ' 'mm_ratio, expected', [ ("binance", False, 'futures', 'isolated', 1535443.01, 0.0, @@ -4531,7 +4569,7 @@ def test_liquidation_price_is_none( ]) def test_liquidation_price( mocker, default_conf, exchange_name, open_rate, is_short, trading_mode, - margin_mode, wallet_balance, mm_ex_1, upnl_ex_1, maintenance_amt, position, mm_ratio, expected + margin_mode, wallet_balance, mm_ex_1, upnl_ex_1, maintenance_amt, amount, mm_ratio, expected ): default_conf['trading_mode'] = trading_mode default_conf['margin_mode'] = margin_mode @@ -4545,7 +4583,8 @@ def test_liquidation_price( wallet_balance=wallet_balance, mm_ex_1=mm_ex_1, upnl_ex_1=upnl_ex_1, - position=position, + amount=amount, + stake_amount=open_rate * amount, ), 2), expected) @@ -4757,6 +4796,20 @@ def test_load_leverage_tiers(mocker, default_conf, leverage_tiers, exchange_name ) +@pytest.mark.asyncio +@pytest.mark.parametrize('exchange_name', EXCHANGES) +async def test_get_market_leverage_tiers(mocker, default_conf, exchange_name): + default_conf['exchange']['name'] = exchange_name + await async_ccxt_exception( + mocker, + default_conf, + MagicMock(), + "get_market_leverage_tiers", + "fetch_market_leverage_tiers", + symbol='BTC/USDT:USDT' + ) + + def test_parse_leverage_tier(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) @@ -4777,10 +4830,10 @@ def test_parse_leverage_tier(mocker, default_conf): } assert exchange.parse_leverage_tier(tier) == { - "min": 0, - "max": 100000, - "mmr": 0.025, - "lev": 20, + "minNotional": 0, + "maxNotional": 100000, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20, "maintAmt": 0.0, } @@ -4806,10 +4859,10 @@ def test_parse_leverage_tier(mocker, default_conf): } assert exchange.parse_leverage_tier(tier2) == { - 'min': 0, - 'max': 2000, - 'mmr': 0.01, - 'lev': 75, + 'minNotional': 0, + 'maxNotional': 2000, + 'maintenanceMarginRate': 0.01, + 'maxLeverage': 75, "maintAmt": None, } @@ -5062,6 +5115,7 @@ def test_get_liquidation_price( pair='ETH/USDT:USDT', open_rate=open_rate, amount=amount, + stake_amount=amount * open_rate / leverage, leverage=leverage, is_short=is_short, ) @@ -5097,7 +5151,7 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange._get_contract_size = MagicMock(return_value=contract_size) + exchange.get_contract_size = MagicMock(return_value=contract_size) api_mock.create_order.reset_mock() order = exchange.stoploss( diff --git a/tests/exchange/test_ftx.py b/tests/exchange/test_ftx.py index 5a83b964a..5213c1b36 100644 --- a/tests/exchange/test_ftx.py +++ b/tests/exchange/test_ftx.py @@ -203,7 +203,7 @@ def test_fetch_stoploss_order_ftx(default_conf, mocker, limit_sell_order, limit_ 'info': { 'orderId': 'mocked_limit_sell', }}]) - api_mock.fetch_order = MagicMock(return_value=limit_sell_order) + api_mock.fetch_order = MagicMock(return_value=limit_sell_order.copy()) # No orderId field - no call to fetch_order resp = exchange.fetch_stoploss_order('X', 'TKN/BTC') @@ -219,11 +219,23 @@ def test_fetch_stoploss_order_ftx(default_conf, mocker, limit_sell_order, limit_ order = {'id': 'X', 'status': 'closed', 'info': {'orderId': None}, 'average': 0.254} api_mock.fetch_orders = MagicMock(return_value=[order]) api_mock.fetch_order.reset_mock() + api_mock.privateGetConditionalOrdersConditionalOrderIdTriggers = MagicMock( + return_value={'result': [ + {'orderId': 'mocked_market_sell', 'type': 'market', 'side': 'sell', 'price': 0.254} + ]}) resp = exchange.fetch_stoploss_order('X', 'TKN/BTC') assert resp # fetch_order not called (no regular order ID) - assert api_mock.fetch_order.call_count == 0 - assert order == order + assert api_mock.fetch_order.call_count == 1 + api_mock.privateGetConditionalOrdersConditionalOrderIdTriggers.call_count == 1 + expected_resp = limit_sell_order.copy() + expected_resp.update({ + 'id_stop': 'X', + 'id': 'X', + 'type': 'stop', + 'status_stop': 'triggered', + }) + assert expected_resp == resp with pytest.raises(InvalidOrderException): api_mock.fetch_orders = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 91c4a3368..b475b84ff 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta, timezone +from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pytest @@ -6,7 +7,7 @@ import pytest from freqtrade.enums import MarginMode, TradingMode from freqtrade.enums.candletype import CandleType from freqtrade.exchange.exchange import timeframe_to_minutes -from tests.conftest import get_mock_coro, get_patched_exchange +from tests.conftest import get_mock_coro, get_patched_exchange, log_has from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -267,7 +268,10 @@ def test_additional_exchange_init_okx(default_conf, mocker): "additional_exchange_init", "fetch_accounts") -def test_load_leverage_tiers_okx(default_conf, mocker, markets): +def test_load_leverage_tiers_okx(default_conf, mocker, markets, tmpdir, caplog, time_machine): + + default_conf['datadir'] = Path(tmpdir) + # fd_mock = mocker.patch('freqtrade.exchange.exchange.file_dump_json') api_mock = MagicMock() type(api_mock).has = PropertyMock(return_value={ 'fetchLeverageTiers': False, @@ -410,48 +414,66 @@ def test_load_leverage_tiers_okx(default_conf, mocker, markets): assert exchange._leverage_tiers == { 'ADA/USDT:USDT': [ { - 'min': 0, - 'max': 500, - 'mmr': 0.02, - 'lev': 75, + 'minNotional': 0, + 'maxNotional': 500, + 'maintenanceMarginRate': 0.02, + 'maxLeverage': 75, 'maintAmt': None }, { - 'min': 501, - 'max': 1000, - 'mmr': 0.025, - 'lev': 50, + 'minNotional': 501, + 'maxNotional': 1000, + 'maintenanceMarginRate': 0.025, + 'maxLeverage': 50, 'maintAmt': None }, { - 'min': 1001, - 'max': 2000, - 'mmr': 0.03, - 'lev': 20, + 'minNotional': 1001, + 'maxNotional': 2000, + 'maintenanceMarginRate': 0.03, + 'maxLeverage': 20, 'maintAmt': None }, ], 'ETH/USDT:USDT': [ { - 'min': 0, - 'max': 2000, - 'mmr': 0.01, - 'lev': 75, + 'minNotional': 0, + 'maxNotional': 2000, + 'maintenanceMarginRate': 0.01, + 'maxLeverage': 75, 'maintAmt': None }, { - 'min': 2001, - 'max': 4000, - 'mmr': 0.015, - 'lev': 50, + 'minNotional': 2001, + 'maxNotional': 4000, + 'maintenanceMarginRate': 0.015, + 'maxLeverage': 50, 'maintAmt': None }, { - 'min': 4001, - 'max': 8000, - 'mmr': 0.02, - 'lev': 20, + 'minNotional': 4001, + 'maxNotional': 8000, + 'maintenanceMarginRate': 0.02, + 'maxLeverage': 20, 'maintAmt': None }, ], } + filename = (default_conf['datadir'] / + f"futures/leverage_tiers_{default_conf['stake_currency']}.json") + assert filename.is_file() + + logmsg = 'Cached leverage tiers are outdated. Will update.' + assert not log_has(logmsg, caplog) + + api_mock.fetch_market_leverage_tiers.reset_mock() + + exchange.load_leverage_tiers() + assert not log_has(logmsg, caplog) + + api_mock.fetch_market_leverage_tiers.call_count == 0 + # 2 day passes ... + time_machine.move_to(datetime.now() + timedelta(days=2)) + exchange.load_leverage_tiers() + + assert log_has(logmsg, caplog) diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py new file mode 100644 index 000000000..dd148da77 --- /dev/null +++ b/tests/freqai/conftest.py @@ -0,0 +1,172 @@ +from copy import deepcopy +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider +from freqtrade.freqai.data_drawer import FreqaiDataDrawer +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.resolvers import StrategyResolver +from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver +from tests.conftest import get_patched_exchange + + +@pytest.fixture(scope="function") +def freqai_conf(default_conf, tmpdir): + freqaiconf = deepcopy(default_conf) + freqaiconf.update( + { + "datadir": Path(default_conf["datadir"]), + "strategy": "freqai_test_strat", + "user_data_dir": Path(tmpdir), + "strategy-path": "freqtrade/tests/strategy/strats", + "freqaimodel": "LightGBMRegressor", + "freqaimodel_path": "freqai/prediction_models", + "timerange": "20180110-20180115", + "freqai": { + "enabled": True, + "startup_candles": 10000, + "purge_old_models": True, + "train_period_days": 5, + "backtest_period_days": 2, + "live_retrain_hours": 0, + "expiration_hours": 1, + "identifier": "uniqe-id100", + "live_trained_timestamp": 0, + "feature_parameters": { + "include_timeframes": ["5m"], + "include_corr_pairlist": ["ADA/BTC", "DASH/BTC"], + "label_period_candles": 20, + "include_shifted_candles": 1, + "DI_threshold": 0.9, + "weight_factor": 0.9, + "principal_component_analysis": False, + "use_SVM_to_remove_outliers": True, + "stratify_training_data": 0, + "indicator_max_period_candles": 10, + "indicator_periods_candles": [10], + }, + "data_split_parameters": {"test_size": 0.33, "random_state": 1}, + "model_training_parameters": {"n_estimators": 100}, + }, + "config_files": [Path('config_examples', 'config_freqai.example.json')] + } + ) + freqaiconf['exchange'].update({'pair_whitelist': ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC']}) + return freqaiconf + + +def get_patched_data_kitchen(mocker, freqaiconf): + dk = FreqaiDataKitchen(freqaiconf) + return dk + + +def get_patched_data_drawer(mocker, freqaiconf): + # dd = mocker.patch('freqtrade.freqai.data_drawer', MagicMock()) + dd = FreqaiDataDrawer(freqaiconf) + return dd + + +def get_patched_freqai_strategy(mocker, freqaiconf): + strategy = StrategyResolver.load_strategy(freqaiconf) + strategy.ft_bot_start() + + return strategy + + +def get_patched_freqaimodel(mocker, freqaiconf): + freqaimodel = FreqaiModelResolver.load_freqaimodel(freqaiconf) + + return freqaimodel + + +def make_data_dictionary(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.pair = "ADA/BTC" + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + corr_dataframes, base_dataframes = freqai.dd.get_base_and_corr_dataframes( + data_load_timerange, freqai.dk.pair, freqai.dk + ) + + unfiltered_dataframe = freqai.dk.use_strategy_to_populate_indicators( + strategy, corr_dataframes, base_dataframes, freqai.dk.pair + ) + + unfiltered_dataframe = freqai.dk.slice_dataframe(new_timerange, unfiltered_dataframe) + + freqai.dk.find_features(unfiltered_dataframe) + + features_filtered, labels_filtered = freqai.dk.filter_features( + unfiltered_dataframe, + freqai.dk.training_features_list, + freqai.dk.label_list, + training_filter=True, + ) + + data_dictionary = freqai.dk.make_train_test_datasets(features_filtered, labels_filtered) + + data_dictionary = freqai.dk.normalize_data(data_dictionary) + + return freqai + + +def get_freqai_live_analyzed_dataframe(mocker, freqaiconf): + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dk.load_all_pair_histories(timerange) + + strategy.analyze_pair('ADA/BTC', '5m') + return strategy.dp.get_analyzed_dataframe('ADA/BTC', '5m') + + +def get_freqai_analyzed_dataframe(mocker, freqaiconf): + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dk.load_all_pair_histories(timerange) + sub_timerange = TimeRange.parse_timerange("20180111-20180114") + corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") + + return freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC') + + +def get_ready_to_train(mocker, freqaiconf): + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dk.load_all_pair_histories(timerange) + sub_timerange = TimeRange.parse_timerange("20180111-20180114") + corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") + return corr_df, base_df, freqai, strategy diff --git a/tests/freqai/test_freqai_backtesting.py b/tests/freqai/test_freqai_backtesting.py new file mode 100644 index 000000000..273791609 --- /dev/null +++ b/tests/freqai/test_freqai_backtesting.py @@ -0,0 +1,57 @@ +from copy import deepcopy +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import PropertyMock + +import pytest + +from freqtrade.commands.optimize_commands import start_backtesting +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.backtesting import Backtesting +from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange, + patched_configuration_load_config_file) + + +def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir): + patch_exchange(mocker) + + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT'])) + # mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + + patched_configuration_load_config_file(mocker, freqai_conf) + + args = [ + 'backtesting', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), + '--timeframe', '1h', + '--strategy-list', CURRENT_TEST_STRATEGY + ] + args = get_args(args) + with pytest.raises(OperationalException, + match=r"You can't use strategy_list and freqai at the same time\."): + start_backtesting(args) + + +def test_freqai_backtest_load_data(freqai_conf, mocker, caplog): + patch_exchange(mocker) + + now = datetime.now(timezone.utc) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT'])) + mocker.patch('freqtrade.optimize.backtesting.history.load_data') + mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now)) + backtesting = Backtesting(deepcopy(freqai_conf)) + backtesting.load_bt_data() + + assert log_has_re('Increasing startup_candle_count for freqai to.*', caplog) + + del freqai_conf['freqai']['startup_candles'] + backtesting = Backtesting(freqai_conf) + with pytest.raises(OperationalException, + match=r'FreqAI backtesting module.*startup_candles in config.'): + backtesting.load_bt_data() + + Backtesting.cleanup() diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py new file mode 100644 index 000000000..a6df60e61 --- /dev/null +++ b/tests/freqai/test_freqai_datadrawer.py @@ -0,0 +1,94 @@ + +import shutil +from pathlib import Path + +from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from tests.conftest import get_patched_exchange +from tests.freqai.conftest import get_patched_freqai_strategy + + +def test_update_historic_data(mocker, freqai_conf): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180114") + + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) + dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m")) + candle_difference = dp_candles - historic_candles + freqai.dd.update_historic_data(strategy, freqai.dk) + + updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) + + assert updated_historic_candles - historic_candles == candle_difference + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_load_all_pairs_histories(mocker, freqai_conf): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + assert len(freqai.dd.historic_data.keys()) == len( + freqai_conf.get("exchange", {}).get("pair_whitelist") + ) + assert len(freqai.dd.historic_data["ADA/BTC"]) == len( + freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes") + ) + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_get_base_and_corr_dataframes(mocker, freqai_conf): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + sub_timerange = TimeRange.parse_timerange("20180111-20180114") + corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) + + num_tfs = len( + freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes") + ) + + assert len(base_df.keys()) == num_tfs + + assert len(corr_df.keys()) == len( + freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_corr_pairlist") + ) + + assert len(corr_df["ADA/BTC"].keys()) == num_tfs + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_use_strategy_to_populate_indicators(mocker, freqai_conf): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + sub_timerange = TimeRange.parse_timerange("20180111-20180114") + corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) + + df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC') + + assert len(df.columns) == 45 + shutil.rmtree(Path(freqai.dk.full_path)) diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py new file mode 100644 index 000000000..9ef955695 --- /dev/null +++ b/tests/freqai/test_freqai_datakitchen.py @@ -0,0 +1,96 @@ +import datetime +import shutil +from pathlib import Path + +import pytest + +from freqtrade.exceptions import OperationalException +from tests.conftest import log_has_re +from tests.freqai.conftest import get_patched_data_kitchen, make_data_dictionary + + +@pytest.mark.parametrize( + "timerange, train_period_days, expected_result", + [ + ("20220101-20220201", 30, "20211202-20220201"), + ("20220301-20220401", 15, "20220214-20220401"), + ], +) +def test_create_fulltimerange( + timerange, train_period_days, expected_result, freqai_conf, mocker, caplog +): + dk = get_patched_data_kitchen(mocker, freqai_conf) + assert dk.create_fulltimerange(timerange, train_period_days) == expected_result + shutil.rmtree(Path(dk.full_path)) + + +def test_create_fulltimerange_incorrect_backtest_period(mocker, freqai_conf): + dk = get_patched_data_kitchen(mocker, freqai_conf) + with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"): + dk.create_fulltimerange("20220101-20220201", 0.5) + with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"): + dk.create_fulltimerange("20220101-20220201", -1) + shutil.rmtree(Path(dk.full_path)) + + +@pytest.mark.parametrize( + "timerange, train_period_days, backtest_period_days, expected_result", + [ + ("20220101-20220201", 30, 7, 9), + ("20220101-20220201", 30, 0.5, 120), + ("20220101-20220201", 10, 1, 80), + ], +) +def test_split_timerange( + mocker, freqai_conf, timerange, train_period_days, backtest_period_days, expected_result +): + freqai_conf.update({"timerange": "20220101-20220401"}) + dk = get_patched_data_kitchen(mocker, freqai_conf) + tr_list, bt_list = dk.split_timerange(timerange, train_period_days, backtest_period_days) + assert len(tr_list) == len(bt_list) == expected_result + + with pytest.raises( + OperationalException, match=r"train_period_days must be an integer greater than 0." + ): + dk.split_timerange("20220101-20220201", -1, 0.5) + shutil.rmtree(Path(dk.full_path)) + + +@pytest.mark.parametrize( + "timestamp, expected", + [ + (datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - 7200, True), + (datetime.datetime.now(tz=datetime.timezone.utc).timestamp(), False), + ], +) +def test_check_if_model_expired(mocker, freqai_conf, timestamp, expected): + dk = get_patched_data_kitchen(mocker, freqai_conf) + assert dk.check_if_model_expired(timestamp) == expected + shutil.rmtree(Path(dk.full_path)) + + +def test_use_DBSCAN_to_remove_outliers(mocker, freqai_conf, caplog): + freqai = make_data_dictionary(mocker, freqai_conf) + # freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 1}) + freqai.dk.use_DBSCAN_to_remove_outliers(predict=False) + assert log_has_re( + "DBSCAN found eps of 2.42.", + caplog, + ) + + +def test_compute_distances(mocker, freqai_conf): + freqai = make_data_dictionary(mocker, freqai_conf) + freqai_conf['freqai']['feature_parameters'].update({"DI_threshold": 1}) + avg_mean_dist = freqai.dk.compute_distances() + assert round(avg_mean_dist, 2) == 2.56 + + +def test_use_SVM_to_remove_outliers_and_outlier_protection(mocker, freqai_conf, caplog): + freqai = make_data_dictionary(mocker, freqai_conf) + freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 0.1}) + freqai.dk.use_SVM_to_remove_outliers(predict=False) + assert log_has_re( + "SVM detected 8.46%", + caplog, + ) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py new file mode 100644 index 000000000..792ffc467 --- /dev/null +++ b/tests/freqai/test_freqai_interface.py @@ -0,0 +1,345 @@ +import platform +import shutil +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from tests.conftest import get_patched_exchange, log_has_re +from tests.freqai.conftest import get_patched_freqai_strategy + + +def is_arm() -> bool: + machine = platform.machine() + return "arm" in machine or "aarch64" in machine + + +def test_train_model_in_series_LightGBM(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file() + + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_train_model_in_series_LightGBMMultiModel(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.update({"strategy": "freqai_test_multimodel_strat"}) + freqai_conf.update({"freqaimodel": "LightGBMRegressorMultiTarget"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + assert len(freqai.dk.label_list) == 2 + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file() + assert len(freqai.dk.data['training_features_list']) == 26 + + shutil.rmtree(Path(freqai.dk.full_path)) + + +@pytest.mark.skipif(is_arm(), reason="no ARM for Catboost ...") +def test_train_model_in_series_Catboost(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.update({"freqaimodel": "CatboostRegressor"}) + # freqai_conf.get('freqai', {}).update( + # {'model_training_parameters': {"n_estimators": 100, "verbose": 0}}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series(new_timerange, "ADA/BTC", + strategy, freqai.dk, data_load_timerange) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").exists() + + shutil.rmtree(Path(freqai.dk.full_path)) + + +@pytest.mark.skipif(is_arm(), reason="no ARM for Catboost ...") +def test_train_model_in_series_CatboostClassifier(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.update({"freqaimodel": "CatboostClassifier"}) + freqai_conf.update({"strategy": "freqai_test_classifier"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series(new_timerange, "ADA/BTC", + strategy, freqai.dk, data_load_timerange) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").exists() + + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_train_model_in_series_LightGBMClassifier(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.update({"freqaimodel": "LightGBMClassifier"}) + freqai_conf.update({"strategy": "freqai_test_classifier"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series(new_timerange, "ADA/BTC", + strategy, freqai.dk, data_load_timerange) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").exists() + + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_start_backtesting(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180120-20180130"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + sub_timerange = TimeRange.parse_timerange("20180110-20180130") + corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) + + df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + + metadata = {"pair": "LTC/BTC"} + freqai.start_backtesting(df, metadata, freqai.dk) + model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] + + assert len(model_folders) == 5 + + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180120-20180124"}) + freqai_conf.get("freqai", {}).update({"backtest_period_days": 0.5}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + sub_timerange = TimeRange.parse_timerange("20180110-20180130") + corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) + + df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + + metadata = {"pair": "LTC/BTC"} + freqai.start_backtesting(df, metadata, freqai.dk) + model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] + assert len(model_folders) == 8 + + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): + freqai_conf.update({"timerange": "20180120-20180130"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + sub_timerange = TimeRange.parse_timerange("20180110-20180130") + corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) + + df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + + metadata = {"pair": "ADA/BTC"} + freqai.start_backtesting(df, metadata, freqai.dk) + model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] + + assert len(model_folders) == 5 + + # without deleting the exiting folder structure, re-run + + freqai_conf.update({"timerange": "20180120-20180130"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + sub_timerange = TimeRange.parse_timerange("20180110-20180130") + corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) + + df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + freqai.start_backtesting(df, metadata, freqai.dk) + + assert log_has_re( + "Found model at ", + caplog, + ) + + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_follow_mode(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + metadata = {"pair": "ADA/BTC"} + freqai.dd.set_pair_dict_info(metadata) + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file() + + # start the follower and ask it to predict on existing files + + freqai_conf.get("freqai", {}).update({"follow_mode": "true"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.live) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + df = strategy.dp.get_pair_dataframe('ADA/BTC', '5m') + freqai.start_live(df, metadata, strategy, freqai.dk) + + assert len(freqai.dk.return_dataframe.index) == 5702 + + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_principal_component_analysis(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( + {"princpial_component_analysis": "true"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_pca_object.pkl") + + shutil.rmtree(Path(freqai.dk.full_path)) diff --git a/tests/leverage/test_interest.py b/tests/leverage/test_interest.py index 6b189ce50..7bdf4c2f0 100644 --- a/tests/leverage/test_interest.py +++ b/tests/leverage/test_interest.py @@ -1,14 +1,14 @@ -from decimal import Decimal from math import isclose import pytest from freqtrade.leverage import interest +from freqtrade.util import FtPrecise -ten_mins = Decimal(1 / 6) -five_hours = Decimal(5.0) -twentyfive_hours = Decimal(25.0) +ten_mins = FtPrecise(1 / 6) +five_hours = FtPrecise(5.0) +twentyfive_hours = FtPrecise(25.0) @pytest.mark.parametrize('exchange,interest_rate,hours,expected', [ @@ -28,11 +28,11 @@ twentyfive_hours = Decimal(25.0) ('ftx', 0.00025, twentyfive_hours, 0.015625), ]) def test_interest(exchange, interest_rate, hours, expected): - borrowed = Decimal(60.0) + borrowed = FtPrecise(60.0) assert isclose(interest( exchange_name=exchange, borrowed=borrowed, - rate=Decimal(interest_rate), + rate=FtPrecise(interest_rate), hours=hours ), expected) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 0b964c54a..368e368c5 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -550,6 +550,7 @@ def test_backtest__enter_trade_futures(default_conf_usdt, fee, mocker) -> None: mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) mocker.patch("freqtrade.exchange.Exchange.get_max_leverage", return_value=100) + mocker.patch("freqtrade.optimize.backtesting.price_to_precision", lambda p, *args: p) patch_exchange(mocker) default_conf_usdt['stake_amount'] = 300 default_conf_usdt['max_open_trades'] = 2 @@ -559,13 +560,13 @@ def test_backtest__enter_trade_futures(default_conf_usdt, fee, mocker) -> None: default_conf_usdt['exchange']['pair_whitelist'] = ['.*'] backtesting = Backtesting(default_conf_usdt) backtesting._set_strategy(backtesting.strategylist[0]) - pair = 'UNITTEST/USDT:USDT' + pair = 'ETH/USDT:USDT' row = [ pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), - 0.001, # Open - 0.0012, # High - 0.00099, # Low - 0.0011, # Close + 0.1, # Open + 0.12, # High + 0.099, # Low + 0.11, # Close 1, # enter_long 0, # exit_long 1, # enter_short @@ -580,8 +581,8 @@ def test_backtest__enter_trade_futures(default_conf_usdt, fee, mocker) -> None: return_value=(0.01, 0.01)) # leverage = 5 - # ep1(trade.open_rate) = 0.001 - # position(trade.amount) = 1500000 + # ep1(trade.open_rate) = 0.1 + # position(trade.amount) = 15000 # stake_amount = 300 -> wb = 300 / 5 = 60 # mmr = 0.01 # cum_b = 0.01 @@ -591,26 +592,26 @@ def test_backtest__enter_trade_futures(default_conf_usdt, fee, mocker) -> None: # Binance, Long # liquidation_price # = ((wb + cum_b) - (side_1 * position * ep1)) / ((position * mmr_b) - (side_1 * position)) - # = ((300 + 0.01) - (1 * 1500000 * 0.001)) / ((1500000 * 0.01) - (1 * 1500000)) + # = ((300 + 0.01) - (1 * 15000 * 0.1)) / ((15000 * 0.01) - (1 * 15000)) # = 0.0008080740740740741 # freqtrade_liquidation_price = liq + (abs(open_rate - liq) * liq_buffer * side_1) - # = 0.0008080740740740741 + ((0.001 - 0.0008080740740740741) * 0.05 * 1) - # = 0.0008176703703703704 + # = 0.08080740740740741 + ((0.1 - 0.08080740740740741) * 0.05 * 1) + # = 0.08176703703703704 trade = backtesting._enter_trade(pair, row=row, direction='long') - assert pytest.approx(trade.liquidation_price) == 0.00081767037 + assert pytest.approx(trade.liquidation_price) == 0.081767037 # Binance, Short # liquidation_price # = ((wb + cum_b) - (side_1 * position * ep1)) / ((position * mmr_b) - (side_1 * position)) - # = ((300 + 0.01) - ((-1) * 1500000 * 0.001)) / ((1500000 * 0.01) - ((-1) * 1500000)) + # = ((300 + 0.01) - ((-1) * 15000 * 0.1)) / ((15000 * 0.01) - ((-1) * 15000)) # = 0.0011881254125412541 # freqtrade_liquidation_price = liq + (abs(open_rate - liq) * liq_buffer * side_1) - # = 0.0011881254125412541 + (abs(0.001 - 0.0011881254125412541) * 0.05 * -1) - # = 0.0011787191419141915 + # = 0.11881254125412541 + (abs(0.1 - 0.11881254125412541) * 0.05 * -1) + # = 0.11787191419141915 trade = backtesting._enter_trade(pair, row=row, direction='short') - assert pytest.approx(trade.liquidation_price) == 0.0011787191 + assert pytest.approx(trade.liquidation_price) == 0.11787191 # Stake-amount too high! mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=600.0) diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index fca9c01b2..71f8cdcea 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -1,8 +1,10 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument from copy import deepcopy +from unittest.mock import MagicMock import pandas as pd +import pytest from arrow import Arrow from freqtrade.configuration import TimeRange @@ -16,6 +18,8 @@ from tests.conftest import patch_exchange def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_exit_signal'] = False mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch('freqtrade.optimize.backtesting.amount_to_contract_precision', + lambda x, *args, **kwargs: round(x, 8)) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) patch_exchange(mocker) @@ -87,3 +91,87 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> assert (round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) or round(ln.iloc[0]["low"], 6) < round( t["close_rate"], 6) < round(ln.iloc[0]["high"], 6)) + + +def test_backtest_position_adjustment_detailed(default_conf, fee, mocker) -> None: + default_conf['use_exit_signal'] = False + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=10) + mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) + patch_exchange(mocker) + default_conf.update({ + "stake_amount": 100.0, + "dry_run_wallet": 1000.0, + "strategy": "StrategyTestV3" + }) + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + pair = 'XRP/USDT' + row = [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), + 2.1, # Open + 2.2, # High + 1.9, # Low + 2.1, # Close + 1, # enter_long + 0, # exit_long + 0, # enter_short + 0, # exit_short + '', # enter_tag + '', # exit_tag + ] + trade = backtesting._enter_trade(pair, row=row, direction='long') + trade.orders[0].close_bt_order(row[0], trade) + assert trade + assert pytest.approx(trade.stake_amount) == 100.0 + assert pytest.approx(trade.amount) == 47.61904762 + assert len(trade.orders) == 1 + backtesting.strategy.adjust_trade_position = MagicMock(return_value=None) + + trade = backtesting._get_adjust_trade_entry_for_candle(trade, row) + assert trade + assert pytest.approx(trade.stake_amount) == 100.0 + assert pytest.approx(trade.amount) == 47.61904762 + assert len(trade.orders) == 1 + # Increase position by 100 + backtesting.strategy.adjust_trade_position = MagicMock(return_value=100) + + trade = backtesting._get_adjust_trade_entry_for_candle(trade, row) + + assert trade + assert pytest.approx(trade.stake_amount) == 200.0 + assert pytest.approx(trade.amount) == 95.23809524 + assert len(trade.orders) == 2 + + # Reduce by more than amount - no change to trade. + backtesting.strategy.adjust_trade_position = MagicMock(return_value=-500) + + trade = backtesting._get_adjust_trade_entry_for_candle(trade, row) + + assert trade + assert pytest.approx(trade.stake_amount) == 200.0 + assert pytest.approx(trade.amount) == 95.23809524 + assert len(trade.orders) == 2 + assert trade.nr_of_successful_entries == 2 + + # Reduce position by 50 + backtesting.strategy.adjust_trade_position = MagicMock(return_value=-100) + trade = backtesting._get_adjust_trade_entry_for_candle(trade, row) + + assert trade + assert pytest.approx(trade.stake_amount) == 100.0 + assert pytest.approx(trade.amount) == 47.61904762 + assert len(trade.orders) == 3 + assert trade.nr_of_successful_entries == 2 + assert trade.nr_of_successful_exits == 1 + + # Adjust below minimum + backtesting.strategy.adjust_trade_position = MagicMock(return_value=-99) + trade = backtesting._get_adjust_trade_entry_for_candle(trade, row) + + assert trade + assert pytest.approx(trade.stake_amount) == 100.0 + assert pytest.approx(trade.amount) == 47.61904762 + assert len(trade.orders) == 3 + assert trade.nr_of_successful_entries == 2 + assert trade.nr_of_successful_exits == 1 diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index c56f405e2..48a0f81cb 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -12,7 +12,7 @@ from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.enums import CandleType, RunMode from freqtrade.exceptions import OperationalException from freqtrade.persistence import Trade -from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver from tests.conftest import (create_mock_trades_usdt, get_patched_exchange, get_patched_freqtradebot, @@ -366,6 +366,9 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "PrecisionFilter"}], + "USDT", ['ETH/USDT', 'NANO/USDT']), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], @@ -735,7 +738,7 @@ def test_PerformanceFilter_lookback(mocker, default_conf_usdt, fee, caplog) -> N with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: create_mock_trades_usdt(fee) pm.refresh_pairlist() - assert pm.whitelist == ['XRP/USDT'] + assert pm.whitelist == ['XRP/USDT', 'NEO/USDT'] assert log_has_re(r'Removing pair .* since .* is below .*', caplog) # Move to "outside" of lookback window, so original sorting is restored. @@ -762,8 +765,8 @@ def test_PerformanceFilter_keep_mid_order(mocker, default_conf_usdt, fee, caplog with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: create_mock_trades_usdt(fee) pm.refresh_pairlist() - assert pm.whitelist == ['XRP/USDT', 'ETC/USDT', 'ETH/USDT', 'LTC/USDT', - 'NEO/USDT', 'TKN/USDT', 'ADA/USDT', ] + assert pm.whitelist == ['XRP/USDT', 'NEO/USDT', 'ETH/USDT', 'LTC/USDT', + 'TKN/USDT', 'ADA/USDT', 'ETC/USDT', ] # assert log_has_re(r'Removing pair .* since .* is below .*', caplog) # Move to "outside" of lookback window, so original sorting is restored. @@ -1282,6 +1285,22 @@ def test_expand_pairlist(wildcardlist, pairs, expected): expand_pairlist(wildcardlist, pairs) else: assert sorted(expand_pairlist(wildcardlist, pairs)) == sorted(expected) + conf = { + 'pairs': wildcardlist, + 'freqai': { + "enabled": True, + "feature_parameters": { + "include_corr_pairlist": [ + "BTC/USDT:USDT", + "XRP/BUSD", + ] + } + } + } + assert sorted(dynamic_expand_pairlist(conf, pairs)) == sorted(expected + [ + "BTC/USDT:USDT", + "XRP/BUSD", + ]) @pytest.mark.parametrize('wildcardlist,pairs,expected', [ diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 4cebb6492..acfe124a8 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -67,6 +67,8 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, trade.close(open_rate * (2 - profit_rate if is_short else profit_rate)) trade.exit_reason = exit_reason + Trade.query.session.add(trade) + Trade.commit() return trade @@ -125,33 +127,33 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog, is_short): assert not log_has_re(message, caplog) caplog.clear() - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, is_short=is_short, - )) + ) assert not freqtrade.protections.global_stop() assert not log_has_re(message, caplog) caplog.clear() # This trade does not count, as it's closed too long ago - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'BCH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=250, min_ago_close=100, is_short=is_short, - )) + ) - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=240, min_ago_close=30, is_short=is_short, - )) + ) # 3 Trades closed - but the 2nd has been closed too long ago. assert not freqtrade.protections.global_stop() assert not log_has_re(message, caplog) caplog.clear() - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'LTC/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=180, min_ago_close=30, is_short=is_short, - )) + ) assert freqtrade.protections.global_stop() assert log_has_re(message, caplog) @@ -186,25 +188,25 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair assert not log_has_re(message, caplog) caplog.clear() - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, profit_rate=0.9, is_short=is_short - )) + ) assert not freqtrade.protections.stop_per_pair(pair) assert not freqtrade.protections.global_stop() assert not log_has_re(message, caplog) caplog.clear() # This trade does not count, as it's closed too long ago - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=250, min_ago_close=100, profit_rate=0.9, is_short=is_short - )) + ) # Trade does not count for per pair stop as it's the wrong pair. - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=240, min_ago_close=30, profit_rate=0.9, is_short=is_short - )) + ) # 3 Trades closed - but the 2nd has been closed too long ago. assert not freqtrade.protections.stop_per_pair(pair) assert freqtrade.protections.global_stop() != only_per_pair @@ -216,10 +218,10 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair caplog.clear() # Trade does not count potentially, as it's in the wrong direction - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=150, min_ago_close=25, profit_rate=0.9, is_short=not is_short - )) + ) freqtrade.protections.stop_per_pair(pair) assert freqtrade.protections.global_stop() != only_per_pair assert PairLocks.is_pair_locked(pair, side=check_side) != (only_per_side and only_per_pair) @@ -231,10 +233,10 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair caplog.clear() # 2nd Trade that counts with correct pair - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=180, min_ago_close=30, profit_rate=0.9, is_short=is_short - )) + ) freqtrade.protections.stop_per_pair(pair) assert freqtrade.protections.global_stop() != only_per_pair @@ -259,20 +261,20 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog): assert not log_has_re(message, caplog) caplog.clear() - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, - )) + ) assert not freqtrade.protections.global_stop() assert freqtrade.protections.stop_per_pair('XRP/BTC') assert PairLocks.is_pair_locked('XRP/BTC') assert not PairLocks.is_global_lock() - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=205, min_ago_close=35, - )) + ) assert not freqtrade.protections.global_stop() assert not PairLocks.is_pair_locked('ETH/BTC') @@ -300,22 +302,24 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog, only_per_side): assert not log_has_re(message, caplog) caplog.clear() - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=800, min_ago_close=450, profit_rate=0.9, - )) + ) + Trade.commit() # Not locked with 1 trade assert not freqtrade.protections.global_stop() assert not freqtrade.protections.stop_per_pair('XRP/BTC') assert not PairLocks.is_pair_locked('XRP/BTC') assert not PairLocks.is_global_lock() - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=200, min_ago_close=120, profit_rate=0.9, - )) + ) + Trade.commit() # Not locked with 1 trade (first trade is outside of lookback_period) assert not freqtrade.protections.global_stop() assert not freqtrade.protections.stop_per_pair('XRP/BTC') @@ -323,18 +327,20 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog, only_per_side): assert not PairLocks.is_global_lock() # Add positive trade - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=20, min_ago_close=10, profit_rate=1.15, is_short=True - )) + ) + Trade.commit() assert freqtrade.protections.stop_per_pair('XRP/BTC') != only_per_side assert not PairLocks.is_pair_locked('XRP/BTC', side='*') assert PairLocks.is_pair_locked('XRP/BTC', side='long') == only_per_side - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, - min_ago_open=110, min_ago_close=20, profit_rate=0.8, - )) + min_ago_open=110, min_ago_close=21, profit_rate=0.8, + ) + Trade.commit() # Locks due to 2nd trade assert freqtrade.protections.global_stop() != only_per_side @@ -342,6 +348,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog, only_per_side): assert PairLocks.is_pair_locked('XRP/BTC', side='long') assert PairLocks.is_pair_locked('XRP/BTC', side='*') != only_per_side assert not PairLocks.is_global_lock() + Trade.commit() @pytest.mark.usefixtures("init_persistence") @@ -360,36 +367,38 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): assert not freqtrade.protections.stop_per_pair('XRP/BTC') caplog.clear() - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, - )) - Trade.query.session.add(generate_mock_trade( + ) + generate_mock_trade( 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, - )) - Trade.query.session.add(generate_mock_trade( + ) + generate_mock_trade( 'NEO/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, - )) + ) + Trade.commit() # No losing trade yet ... so max_drawdown will raise exception assert not freqtrade.protections.global_stop() assert not freqtrade.protections.stop_per_pair('XRP/BTC') - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=500, min_ago_close=400, profit_rate=0.9, - )) + ) # Not locked with one trade assert not freqtrade.protections.global_stop() assert not freqtrade.protections.stop_per_pair('XRP/BTC') assert not PairLocks.is_pair_locked('XRP/BTC') assert not PairLocks.is_global_lock() - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1200, min_ago_close=1100, profit_rate=0.5, - )) + ) + Trade.commit() # Not locked with 1 trade (2nd trade is outside of lookback_period) assert not freqtrade.protections.global_stop() @@ -399,20 +408,22 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): assert not log_has_re(message, caplog) # Winning trade ... (should not lock, does not change drawdown!) - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=320, min_ago_close=410, profit_rate=1.5, - )) + ) + Trade.commit() assert not freqtrade.protections.global_stop() assert not PairLocks.is_global_lock() caplog.clear() # Add additional negative trade, causing a loss of > 15% - Trade.query.session.add(generate_mock_trade( + generate_mock_trade( 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=20, min_ago_close=10, profit_rate=0.8, - )) + ) + Trade.commit() assert not freqtrade.protections.stop_per_pair('XRP/BTC') # local lock not supported assert not PairLocks.is_pair_locked('XRP/BTC') diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 6e19fcaf3..8bbf75a32 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -96,21 +96,22 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'profit_pct': -0.41, 'profit_abs': -4.09e-06, 'profit_fiat': ANY, - 'stop_loss_abs': 9.882e-06, + 'stop_loss_abs': 9.89e-06, 'stop_loss_pct': -10.0, 'stop_loss_ratio': -0.1, 'stoploss_order_id': None, 'stoploss_last_update': ANY, 'stoploss_last_update_timestamp': ANY, - 'initial_stop_loss_abs': 9.882e-06, + 'initial_stop_loss_abs': 9.89e-06, 'initial_stop_loss_pct': -10.0, 'initial_stop_loss_ratio': -0.1, - 'stoploss_current_dist': -1.1080000000000002e-06, - 'stoploss_current_dist_ratio': -0.10081893, - 'stoploss_current_dist_pct': -10.08, - 'stoploss_entry_dist': -0.00010475, - 'stoploss_entry_dist_ratio': -0.10448878, + 'stoploss_current_dist': pytest.approx(-1.0999999e-06), + 'stoploss_current_dist_ratio': -0.10009099, + 'stoploss_current_dist_pct': -10.01, + 'stoploss_entry_dist': -0.00010402, + 'stoploss_entry_dist_ratio': -0.10376381, 'open_order': None, + 'realized_profit': 0.0, 'exchange': 'binance', 'leverage': 1.0, 'interest_rate': 0.0, @@ -180,22 +181,23 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'profit_pct': ANY, 'profit_abs': ANY, 'profit_fiat': ANY, - 'stop_loss_abs': 9.882e-06, + 'stop_loss_abs': 9.89e-06, 'stop_loss_pct': -10.0, 'stop_loss_ratio': -0.1, 'stoploss_order_id': None, 'stoploss_last_update': ANY, 'stoploss_last_update_timestamp': ANY, - 'initial_stop_loss_abs': 9.882e-06, + 'initial_stop_loss_abs': 9.89e-06, 'initial_stop_loss_pct': -10.0, 'initial_stop_loss_ratio': -0.1, 'stoploss_current_dist': ANY, 'stoploss_current_dist_ratio': ANY, 'stoploss_current_dist_pct': ANY, - 'stoploss_entry_dist': -0.00010475, - 'stoploss_entry_dist_ratio': -0.10448878, + 'stoploss_entry_dist': -0.00010402, + 'stoploss_entry_dist_ratio': -0.10376381, 'open_order': None, 'exchange': 'binance', + 'realized_profit': 0.0, 'leverage': 1.0, 'interest_rate': 0.0, 'liquidation_price': None, @@ -312,10 +314,10 @@ def test__rpc_timeunit_profit(default_conf_usdt, ticker, fee, # {'date': datetime.date(2022, 6, 11), 'abs_profit': 13.8299999, # 'starting_balance': 1055.37, 'rel_profit': 0.0131044, # 'fiat_value': 0.0, 'trade_count': 2} - assert day['abs_profit'] in (0.0, pytest.approx(13.8299999), pytest.approx(-4.0)) - assert day['rel_profit'] in (0.0, pytest.approx(0.01310441), pytest.approx(-0.00377583)) + assert day['abs_profit'] in (0.0, pytest.approx(6.83), pytest.approx(-4.09)) + assert day['rel_profit'] in (0.0, pytest.approx(0.00642902), pytest.approx(-0.00383512)) assert day['trade_count'] in (0, 1, 2) - assert day['starting_balance'] in (pytest.approx(1059.37), pytest.approx(1055.37)) + assert day['starting_balance'] in (pytest.approx(1062.37), pytest.approx(1066.46)) assert day['fiat_value'] in (0.0, ) # ensure first day is current date assert str(days['data'][0]['date']) == str(datetime.utcnow().date()) @@ -433,9 +435,9 @@ def test_rpc_trade_statistics(default_conf_usdt, ticker, fee, mocker) -> None: create_mock_trades_usdt(fee) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) - assert pytest.approx(stats['profit_closed_coin']) == 9.83 + assert pytest.approx(stats['profit_closed_coin']) == 2.74 assert pytest.approx(stats['profit_closed_percent_mean']) == -1.67 - assert pytest.approx(stats['profit_closed_fiat']) == 10.813 + assert pytest.approx(stats['profit_closed_fiat']) == 3.014 assert pytest.approx(stats['profit_all_coin']) == -77.45964918 assert pytest.approx(stats['profit_all_percent_mean']) == -57.86 assert pytest.approx(stats['profit_all_fiat']) == -85.205614098 @@ -459,46 +461,6 @@ def test_rpc_trade_statistics(default_conf_usdt, ticker, fee, mocker) -> None: assert isnan(stats['profit_all_coin']) -# Test that rpc_trade_statistics can handle trades that lacks -# trade.open_rate (it is set to None) -def test_rpc_trade_statistics_closed(mocker, default_conf_usdt, ticker, fee): - mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', - return_value=1.1) - mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - get_fee=fee, - ) - - freqtradebot = get_patched_freqtradebot(mocker, default_conf_usdt) - patch_get_signal(freqtradebot) - stake_currency = default_conf_usdt['stake_currency'] - fiat_display_currency = default_conf_usdt['fiat_display_currency'] - - rpc = RPC(freqtradebot) - - # Create some test data - create_mock_trades_usdt(fee) - - for trade in Trade.query.order_by(Trade.id).all(): - trade.open_rate = None - - stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) - assert stats['profit_closed_coin'] == 0 - assert stats['profit_closed_percent_mean'] == 0 - assert stats['profit_closed_fiat'] == 0 - assert stats['profit_all_coin'] == 0 - assert stats['profit_all_percent_mean'] == 0 - assert stats['profit_all_fiat'] == 0 - assert stats['trade_count'] == 7 - assert stats['first_trade_date'] == '2 days ago' - assert stats['latest_trade_date'] == '17 minutes ago' - assert stats['avg_duration'] == '0:00:00' - assert stats['best_pair'] == 'XRP/USDT' - assert stats['best_rate'] == 10.0 - - def test_rpc_balance_handle_error(default_conf, mocker): mock_balance = { 'BTC': { @@ -701,7 +663,7 @@ def test_rpc_stop(mocker, default_conf) -> None: assert freqtradebot.state == State.STOPPED -def test_rpc_stopbuy(mocker, default_conf) -> None: +def test_rpc_stopentry(mocker, default_conf) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -714,8 +676,8 @@ def test_rpc_stopbuy(mocker, default_conf) -> None: freqtradebot.state = State.RUNNING assert freqtradebot.config['max_open_trades'] != 0 - result = rpc._rpc_stopbuy() - assert {'status': 'No more buy will occur from now. Run /reload_config to reset.'} == result + result = rpc._rpc_stopentry() + assert {'status': 'No more entries will occur from now. Run /reload_config to reset.'} == result assert freqtradebot.config['max_open_trades'] == 0 @@ -799,7 +761,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: # and trade amount is updated rpc._rpc_force_exit('3') assert cancel_order_mock.call_count == 1 - assert trade.amount == filled_amount + assert pytest.approx(trade.amount) == filled_amount mocker.patch( 'freqtrade.exchange.Exchange.fetch_order', @@ -841,7 +803,8 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: 'side': 'sell', 'amount': amount, 'remaining': amount, - 'filled': 0.0 + 'filled': 0.0, + 'id': trade.orders[0].order_id, } ) msg = rpc._rpc_force_exit('3') @@ -867,9 +830,9 @@ def test_performance_handle(default_conf_usdt, ticker, fee, mocker) -> None: res = rpc._rpc_performance() assert len(res) == 3 - assert res[0]['pair'] == 'XRP/USDT' + assert res[0]['pair'] == 'NEO/USDT' assert res[0]['count'] == 1 - assert res[0]['profit_pct'] == 10.0 + assert res[0]['profit_pct'] == 5.0 def test_enter_tag_performance_handle(default_conf, ticker, fee, mocker) -> None: @@ -893,16 +856,16 @@ def test_enter_tag_performance_handle(default_conf, ticker, fee, mocker) -> None res = rpc._rpc_enter_tag_performance(None) assert len(res) == 3 - assert res[0]['enter_tag'] == 'TEST3' + assert res[0]['enter_tag'] == 'TEST1' assert res[0]['count'] == 1 - assert res[0]['profit_pct'] == 10.0 + assert res[0]['profit_pct'] == 5.0 res = rpc._rpc_enter_tag_performance(None) assert len(res) == 3 - assert res[0]['enter_tag'] == 'TEST3' + assert res[0]['enter_tag'] == 'TEST1' assert res[0]['count'] == 1 - assert res[0]['profit_pct'] == 10.0 + assert res[0]['profit_pct'] == 5.0 def test_enter_tag_performance_handle_2(mocker, default_conf, markets, fee): @@ -953,11 +916,11 @@ def test_exit_reason_performance_handle(default_conf_usdt, ticker, fee, mocker) res = rpc._rpc_exit_reason_performance(None) assert len(res) == 3 - assert res[0]['exit_reason'] == 'roi' + assert res[0]['exit_reason'] == 'exit_signal' assert res[0]['count'] == 1 - assert res[0]['profit_pct'] == 10.0 + assert res[0]['profit_pct'] == 5.0 - assert res[1]['exit_reason'] == 'exit_signal' + assert res[1]['exit_reason'] == 'roi' assert res[2]['exit_reason'] == 'Other' @@ -1009,9 +972,9 @@ def test_mix_tag_performance_handle(default_conf, ticker, fee, mocker) -> None: res = rpc._rpc_mix_tag_performance(None) assert len(res) == 3 - assert res[0]['mix_tag'] == 'TEST3 roi' + assert res[0]['mix_tag'] == 'TEST1 exit_signal' assert res[0]['count'] == 1 - assert res[0]['profit_pct'] == 10.0 + assert res[0]['profit_pct'] == 5.0 def test_mix_tag_performance_handle_2(mocker, default_conf, markets, fee): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 57ba8e9f1..5dfa77d8b 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -109,6 +109,9 @@ def test_api_ui_fallback(botclient, mocker): rc = client_get(client, "/something") assert rc.status_code == 200 + rc = client_get(client, "/something.js") + assert rc.status_code == 200 + # Test directory traversal without mock rc = client_get(client, '%2F%2F%2Fetc/passwd') assert rc.status_code == 200 @@ -419,13 +422,20 @@ def test_api_reloadconf(botclient): assert ftbot.state == State.RELOAD_CONFIG -def test_api_stopbuy(botclient): +def test_api_stopentry(botclient): ftbot, client = botclient assert ftbot.config['max_open_trades'] != 0 rc = client_post(client, f"{BASE_URI}/stopbuy") assert_response(rc) - assert rc.json() == {'status': 'No more buy will occur from now. Run /reload_config to reset.'} + assert rc.json() == { + 'status': 'No more entries will occur from now. Run /reload_config to reset.'} + assert ftbot.config['max_open_trades'] == 0 + + rc = client_post(client, f"{BASE_URI}/stopentry") + assert_response(rc) + assert rc.json() == { + 'status': 'No more entries will occur from now. Run /reload_config to reset.'} assert ftbot.config['max_open_trades'] == 0 @@ -717,11 +727,11 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): ( True, {'best_pair': 'ETC/BTC', 'best_rate': -0.5, 'best_pair_profit_ratio': -0.005, - 'profit_all_coin': 43.61269123, - 'profit_all_fiat': 538398.67323435, 'profit_all_percent_mean': 66.41, + 'profit_all_coin': 45.561959, + 'profit_all_fiat': 562462.39126200, 'profit_all_percent_mean': 66.41, 'profit_all_ratio_mean': 0.664109545, 'profit_all_percent_sum': 398.47, - 'profit_all_ratio_sum': 3.98465727, 'profit_all_percent': 4.36, - 'profit_all_ratio': 0.043612222872799825, 'profit_closed_coin': -0.00673913, + 'profit_all_ratio_sum': 3.98465727, 'profit_all_percent': 4.56, + 'profit_all_ratio': 0.04556147, 'profit_closed_coin': -0.00673913, 'profit_closed_fiat': -83.19455985, 'profit_closed_ratio_mean': -0.0075, 'profit_closed_percent_mean': -0.75, 'profit_closed_ratio_sum': -0.015, 'profit_closed_percent_sum': -1.5, 'profit_closed_ratio': -6.739057628404269e-06, @@ -732,11 +742,11 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): ( False, {'best_pair': 'XRP/BTC', 'best_rate': 1.0, 'best_pair_profit_ratio': 0.01, - 'profit_all_coin': -44.0631579, - 'profit_all_fiat': -543959.6842755, 'profit_all_percent_mean': -66.41, + 'profit_all_coin': -45.79641127, + 'profit_all_fiat': -565356.69712815, 'profit_all_percent_mean': -66.41, 'profit_all_ratio_mean': -0.6641100666666667, 'profit_all_percent_sum': -398.47, - 'profit_all_ratio_sum': -3.9846604, 'profit_all_percent': -4.41, - 'profit_all_ratio': -0.044063014216106644, 'profit_closed_coin': 0.00073913, + 'profit_all_ratio_sum': -3.9846604, 'profit_all_percent': -4.58, + 'profit_all_ratio': -0.045796261934205953, 'profit_closed_coin': 0.00073913, 'profit_closed_fiat': 9.124559849999999, 'profit_closed_ratio_mean': 0.0075, 'profit_closed_percent_mean': 0.75, 'profit_closed_ratio_sum': 0.015, 'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07, @@ -747,11 +757,11 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): ( None, {'best_pair': 'XRP/BTC', 'best_rate': 1.0, 'best_pair_profit_ratio': 0.01, - 'profit_all_coin': -14.43790415, - 'profit_all_fiat': -178235.92673175, 'profit_all_percent_mean': 0.08, + 'profit_all_coin': -14.94732578, + 'profit_all_fiat': -184524.7367541, 'profit_all_percent_mean': 0.08, 'profit_all_ratio_mean': 0.000835751666666662, 'profit_all_percent_sum': 0.5, - 'profit_all_ratio_sum': 0.005014509999999972, 'profit_all_percent': -1.44, - 'profit_all_ratio': -0.014437768014451796, 'profit_closed_coin': -0.00542913, + 'profit_all_ratio_sum': 0.005014509999999972, 'profit_all_percent': -1.49, + 'profit_all_ratio': -0.014947184841095841, 'profit_closed_coin': -0.00542913, 'profit_closed_fiat': -67.02260985, 'profit_closed_ratio_mean': 0.0025, 'profit_closed_percent_mean': 0.25, 'profit_closed_ratio_sum': 0.005, 'profit_closed_percent_sum': 0.5, 'profit_closed_ratio': -5.429078808526421e-06, @@ -790,22 +800,22 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected) 'first_trade_timestamp': ANY, 'latest_trade_date': '5 minutes ago', 'latest_trade_timestamp': ANY, - 'profit_all_coin': expected['profit_all_coin'], - 'profit_all_fiat': expected['profit_all_fiat'], - 'profit_all_percent_mean': expected['profit_all_percent_mean'], - 'profit_all_ratio_mean': expected['profit_all_ratio_mean'], - 'profit_all_percent_sum': expected['profit_all_percent_sum'], - 'profit_all_ratio_sum': expected['profit_all_ratio_sum'], - 'profit_all_percent': expected['profit_all_percent'], - 'profit_all_ratio': expected['profit_all_ratio'], - 'profit_closed_coin': expected['profit_closed_coin'], - 'profit_closed_fiat': expected['profit_closed_fiat'], - 'profit_closed_ratio_mean': expected['profit_closed_ratio_mean'], - 'profit_closed_percent_mean': expected['profit_closed_percent_mean'], - 'profit_closed_ratio_sum': expected['profit_closed_ratio_sum'], - 'profit_closed_percent_sum': expected['profit_closed_percent_sum'], - 'profit_closed_ratio': expected['profit_closed_ratio'], - 'profit_closed_percent': expected['profit_closed_percent'], + 'profit_all_coin': pytest.approx(expected['profit_all_coin']), + 'profit_all_fiat': pytest.approx(expected['profit_all_fiat']), + 'profit_all_percent_mean': pytest.approx(expected['profit_all_percent_mean']), + 'profit_all_ratio_mean': pytest.approx(expected['profit_all_ratio_mean']), + 'profit_all_percent_sum': pytest.approx(expected['profit_all_percent_sum']), + 'profit_all_ratio_sum': pytest.approx(expected['profit_all_ratio_sum']), + 'profit_all_percent': pytest.approx(expected['profit_all_percent']), + 'profit_all_ratio': pytest.approx(expected['profit_all_ratio']), + 'profit_closed_coin': pytest.approx(expected['profit_closed_coin']), + 'profit_closed_fiat': pytest.approx(expected['profit_closed_fiat']), + 'profit_closed_ratio_mean': pytest.approx(expected['profit_closed_ratio_mean']), + 'profit_closed_percent_mean': pytest.approx(expected['profit_closed_percent_mean']), + 'profit_closed_ratio_sum': pytest.approx(expected['profit_closed_ratio_sum']), + 'profit_closed_percent_sum': pytest.approx(expected['profit_closed_percent_sum']), + 'profit_closed_ratio': pytest.approx(expected['profit_closed_ratio']), + 'profit_closed_percent': pytest.approx(expected['profit_closed_percent']), 'trade_count': 6, 'closed_trade_count': 2, 'winning_trades': expected['winning_trades'], @@ -889,7 +899,7 @@ def test_api_performance(botclient, fee): assert_response(rc) assert len(rc.json()) == 2 assert rc.json() == [{'count': 1, 'pair': 'LTC/ETH', 'profit': 7.61, 'profit_pct': 7.61, - 'profit_ratio': 0.07609203, 'profit_abs': 0.01872279}, + 'profit_ratio': 0.07609203, 'profit_abs': 0.0187228}, {'count': 1, 'pair': 'XRP/ETH', 'profit': -5.57, 'profit_pct': -5.57, 'profit_ratio': -0.05570419, 'profit_abs': -0.1150375}] @@ -1202,7 +1212,7 @@ def test_api_forceexit(botclient, mocker, ticker, fee, markets): fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets), - _is_dry_limit_order_filled=MagicMock(return_value=False), + _is_dry_limit_order_filled=MagicMock(return_value=True), ) patch_get_signal(ftbot) @@ -1212,12 +1222,27 @@ def test_api_forceexit(botclient, mocker, ticker, fee, markets): assert rc.json() == {"error": "Error querying /api/v1/forceexit: invalid argument"} Trade.query.session.rollback() - ftbot.enter_positions() + create_mock_trades(fee) + trade = Trade.get_trades([Trade.id == 5]).first() + assert pytest.approx(trade.amount) == 123 + rc = client_post(client, f"{BASE_URI}/forceexit", + data='{"tradeid": "5", "ordertype": "market", "amount": 23}') + assert_response(rc) + assert rc.json() == {'result': 'Created sell order for trade 5.'} + Trade.query.session.rollback() + + trade = Trade.get_trades([Trade.id == 5]).first() + assert pytest.approx(trade.amount) == 100 + assert trade.is_open is True rc = client_post(client, f"{BASE_URI}/forceexit", - data='{"tradeid": "1"}') + data='{"tradeid": "5"}') assert_response(rc) - assert rc.json() == {'result': 'Created sell order for trade 1.'} + assert rc.json() == {'result': 'Created sell order for trade 5.'} + Trade.query.session.rollback() + + trade = Trade.get_trades([Trade.id == 5]).first() + assert trade.is_open is False def test_api_pair_candles(botclient, ohlcv_history): @@ -1402,7 +1427,10 @@ def test_api_strategies(botclient): 'InformativeDecoratorTest', 'StrategyTestV2', 'StrategyTestV3', - 'StrategyTestV3Futures' + 'StrategyTestV3Futures', + 'freqai_test_classifier', + 'freqai_test_multimodel_strat', + 'freqai_test_strat' ]} diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index 596b5ae20..b9ae16a20 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, C0103 import logging import time +from collections import deque from unittest.mock import MagicMock from freqtrade.enums import RPCMessageType @@ -81,9 +82,25 @@ def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: assert telegram_mock.call_count == 0 +def test_process_msg_queue(mocker, default_conf, caplog) -> None: + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg') + mocker.patch('freqtrade.rpc.telegram.Telegram._init') + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + rpc_manager = RPCManager(freqtradebot) + queue = deque() + queue.append('Test message') + queue.append('Test message 2') + rpc_manager.process_msg_queue(queue) + + assert log_has("Sending rpc message: {'type': strategy_msg, 'msg': 'Test message'}", caplog) + assert log_has("Sending rpc message: {'type': strategy_msg, 'msg': 'Test message 2'}", caplog) + assert telegram_mock.call_count == 2 + + def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: - telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) - mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg') + mocker.patch('freqtrade.rpc.telegram.Telegram._init') freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index f69b7e878..cde7025a7 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -103,7 +103,8 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: "['stats'], ['daily'], ['weekly'], ['monthly'], " "['count'], ['locks'], ['unlock', 'delete_locks'], " "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], " - "['stopbuy'], ['whitelist'], ['blacklist'], ['blacklist_delete', 'bl_delete'], " + "['stopbuy', 'stopentry'], ['whitelist'], ['blacklist'], " + "['blacklist_delete', 'bl_delete'], " "['logs'], ['edge'], ['health'], ['help'], ['version']" "]") @@ -272,7 +273,7 @@ def test_telegram_status_multi_entry(default_conf, update, mocker, fee) -> None: msg = msg_mock.call_args_list[0][0][0] assert re.search(r'Number of Entries.*2', msg) assert re.search(r'Average Entry Price', msg) - assert re.search(r'Order filled at', msg) + assert re.search(r'Order filled', msg) assert re.search(r'Close Date:', msg) is None assert re.search(r'Close Profit:', msg) is None @@ -342,7 +343,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: # close_rate should not be included in the message as the trade is not closed # and no line should be empty lines = msg_mock.call_args_list[0][0][0].split('\n') - assert '' not in lines + assert '' not in lines[:-1] assert 'Close Rate' not in ''.join(lines) assert 'Close Profit' not in ''.join(lines) @@ -357,13 +358,29 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: telegram._status(update=update, context=context) lines = msg_mock.call_args_list[0][0][0].split('\n') - assert '' not in lines + assert '' not in lines[:-1] assert 'Close Rate' not in ''.join(lines) assert 'Close Profit' not in ''.join(lines) assert msg_mock.call_count == 2 assert 'LTC/BTC' in msg_mock.call_args_list[0][0][0] + mocker.patch('freqtrade.rpc.telegram.MAX_MESSAGE_LENGTH', 500) + + msg_mock.reset_mock() + context = MagicMock() + context.args = ["2"] + telegram._status(update=update, context=context) + + assert msg_mock.call_count == 2 + + msg1 = msg_mock.call_args_list[0][0][0] + msg2 = msg_mock.call_args_list[1][0][0] + + assert 'Close Rate' not in msg1 + assert 'Trade ID:* `2`' in msg1 + assert 'Trade ID:* `2` - continued' in msg2 + def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( @@ -433,10 +450,10 @@ def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker, time_machi assert "Daily Profit over the last 2 days:" in msg_mock.call_args_list[0][0][0] assert 'Day ' in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] - assert ' 13.83 USDT' in msg_mock.call_args_list[0][0][0] - assert ' 15.21 USD' in msg_mock.call_args_list[0][0][0] + assert ' 6.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 7.51 USD' in msg_mock.call_args_list[0][0][0] assert '(2)' in msg_mock.call_args_list[0][0][0] - assert '(2) 13.83 USDT 15.21 USD 1.31%' in msg_mock.call_args_list[0][0][0] + assert '(2) 6.83 USDT 7.51 USD 0.64%' in msg_mock.call_args_list[0][0][0] assert '(0)' in msg_mock.call_args_list[0][0][0] # Reset msg_mock @@ -447,8 +464,8 @@ def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker, time_machi assert "Daily Profit over the last 7 days:" in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] assert str((datetime.utcnow() - timedelta(days=5)).date()) in msg_mock.call_args_list[0][0][0] - assert ' 13.83 USDT' in msg_mock.call_args_list[0][0][0] - assert ' 15.21 USD' in msg_mock.call_args_list[0][0][0] + assert ' 6.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 7.51 USD' in msg_mock.call_args_list[0][0][0] assert '(2)' in msg_mock.call_args_list[0][0][0] assert '(1)' in msg_mock.call_args_list[0][0][0] assert '(0)' in msg_mock.call_args_list[0][0][0] @@ -460,8 +477,8 @@ def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker, time_machi context = MagicMock() context.args = ["1"] telegram._daily(update=update, context=context) - assert ' 13.83 USDT' in msg_mock.call_args_list[0][0][0] - assert ' 15.21 USD' in msg_mock.call_args_list[0][0][0] + assert ' 6.83 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 7.51 USD' in msg_mock.call_args_list[0][0][0] assert '(2)' in msg_mock.call_args_list[0][0][0] @@ -523,8 +540,8 @@ def test_weekly_handle(default_conf_usdt, update, ticker, fee, mocker, time_mach today = datetime.utcnow().date() first_iso_day_of_current_week = today - timedelta(days=today.weekday()) assert str(first_iso_day_of_current_week) in msg_mock.call_args_list[0][0][0] - assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] - assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] + assert ' 2.74 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 3.01 USD' in msg_mock.call_args_list[0][0][0] assert '(3)' in msg_mock.call_args_list[0][0][0] assert '(0)' in msg_mock.call_args_list[0][0][0] @@ -536,8 +553,8 @@ def test_weekly_handle(default_conf_usdt, update, ticker, fee, mocker, time_mach assert "Weekly Profit over the last 8 weeks (starting from Monday):" \ in msg_mock.call_args_list[0][0][0] assert 'Weekly' in msg_mock.call_args_list[0][0][0] - assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] - assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] + assert ' 2.74 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 3.01 USD' in msg_mock.call_args_list[0][0][0] assert '(3)' in msg_mock.call_args_list[0][0][0] assert '(0)' in msg_mock.call_args_list[0][0][0] @@ -592,8 +609,8 @@ def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker, time_mac today = datetime.utcnow().date() current_month = f"{today.year}-{today.month:02} " assert current_month in msg_mock.call_args_list[0][0][0] - assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] - assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] + assert ' 2.74 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 3.01 USD' in msg_mock.call_args_list[0][0][0] assert '(3)' in msg_mock.call_args_list[0][0][0] assert '(0)' in msg_mock.call_args_list[0][0][0] @@ -606,8 +623,8 @@ def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker, time_mac assert 'Monthly Profit over the last 6 months:' in msg_mock.call_args_list[0][0][0] assert 'Month ' in msg_mock.call_args_list[0][0][0] assert current_month in msg_mock.call_args_list[0][0][0] - assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] - assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] + assert ' 2.74 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 3.01 USD' in msg_mock.call_args_list[0][0][0] assert '(3)' in msg_mock.call_args_list[0][0][0] assert '(0)' in msg_mock.call_args_list[0][0][0] @@ -620,8 +637,8 @@ def test_monthly_handle(default_conf_usdt, update, ticker, fee, mocker, time_mac telegram._monthly(update=update, context=context) assert msg_mock.call_count == 1 assert 'Monthly Profit over the last 12 months:' in msg_mock.call_args_list[0][0][0] - assert ' 9.83 USDT' in msg_mock.call_args_list[0][0][0] - assert ' 10.81 USD' in msg_mock.call_args_list[0][0][0] + assert ' 2.74 USDT' in msg_mock.call_args_list[0][0][0] + assert ' 3.01 USD' in msg_mock.call_args_list[0][0][0] assert '(3)' in msg_mock.call_args_list[0][0][0] # The one-digit months should contain a zero, Eg: September 2021 = "2021-09" @@ -880,10 +897,10 @@ def test_stopbuy_handle(default_conf, update, mocker) -> None: telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) assert freqtradebot.config['max_open_trades'] != 0 - telegram._stopbuy(update=update, context=MagicMock()) + telegram._stopentry(update=update, context=MagicMock()) assert freqtradebot.config['max_open_trades'] == 0 assert msg_mock.call_count == 1 - assert 'No more buy will occur from now. Run /reload_config to reset.' \ + assert 'No more entries will occur from now. Run /reload_config to reset.' \ in msg_mock.call_args_list[0][0][0] @@ -959,6 +976,9 @@ def test_telegram_forceexit_handle(default_conf, update, ticker, fee, 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY, + 'stake_amount': 0.0009999999999054, + 'sub_trade': False, + 'cumulative_profit': 0.0, } == last_msg @@ -1028,6 +1048,9 @@ def test_telegram_force_exit_down_handle(default_conf, update, ticker, fee, 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY, + 'stake_amount': 0.0009999999999054, + 'sub_trade': False, + 'cumulative_profit': 0.0, } == last_msg @@ -1087,6 +1110,9 @@ def test_forceexit_all_handle(default_conf, update, ticker, fee, mocker) -> None 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY, + 'stake_amount': 0.0009999999999054, + 'sub_trade': False, + 'cumulative_profit': 0.0, } == msg @@ -1259,7 +1285,7 @@ def test_telegram_performance_handle(default_conf_usdt, update, ticker, fee, moc telegram._performance(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'Performance' in msg_mock.call_args_list[0][0][0] - assert 'XRP/USDT\t9.842 USDT (10.00%) (1)' in msg_mock.call_args_list[0][0][0] + assert 'XRP/USDT\t2.842 USDT (10.00%) (1)' in msg_mock.call_args_list[0][0][0] def test_telegram_entry_tag_performance_handle( @@ -1309,7 +1335,7 @@ def test_telegram_exit_reason_performance_handle(default_conf_usdt, update, tick telegram._exit_reason_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Exit Reason Performance' in msg_mock.call_args_list[0][0][0] - assert 'roi\t9.842 USDT (10.00%) (1)' in msg_mock.call_args_list[0][0][0] + assert 'roi\t2.842 USDT (10.00%) (1)' in msg_mock.call_args_list[0][0][0] context.args = ['XRP/USDT'] telegram._exit_reason_performance(update=update, context=context) @@ -1341,7 +1367,7 @@ def test_telegram_mix_tag_performance_handle(default_conf_usdt, update, ticker, telegram._mix_tag_performance(update=update, context=context) assert msg_mock.call_count == 1 assert 'Mix Tag Performance' in msg_mock.call_args_list[0][0][0] - assert ('TEST3 roi\t9.842 USDT (10.00%) (1)' + assert ('TEST3 roi\t2.842 USDT (10.00%) (1)' in msg_mock.call_args_list[0][0][0]) context.args = ['XRP/USDT'] @@ -1433,11 +1459,32 @@ def test_whitelist_static(default_conf, update, mocker) -> None: assert ("Using whitelist `['StaticPairList']` with 4 pairs\n" "`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`" in msg_mock.call_args_list[0][0][0]) + context = MagicMock() + context.args = ['sorted'] + msg_mock.reset_mock() + telegram._whitelist(update=update, context=context) + assert ("Using whitelist `['StaticPairList']` with 4 pairs\n" + "`ETH/BTC, LTC/BTC, NEO/BTC, XRP/BTC`" in msg_mock.call_args_list[0][0][0]) + + context = MagicMock() + context.args = ['baseonly'] + msg_mock.reset_mock() + telegram._whitelist(update=update, context=context) + assert ("Using whitelist `['StaticPairList']` with 4 pairs\n" + "`ETH, LTC, XRP, NEO`" in msg_mock.call_args_list[0][0][0]) + + context = MagicMock() + context.args = ['baseonly', 'sorted'] + msg_mock.reset_mock() + telegram._whitelist(update=update, context=context) + assert ("Using whitelist `['StaticPairList']` with 4 pairs\n" + "`ETH, LTC, NEO, XRP`" in msg_mock.call_args_list[0][0][0]) + def test_whitelist_dynamic(default_conf, update, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) default_conf['pairlists'] = [{'method': 'VolumePairList', - 'number_assets': 4 + 'number_assets': 4 }] telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) @@ -1446,6 +1493,27 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: assert ("Using whitelist `['VolumePairList']` with 4 pairs\n" "`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`" in msg_mock.call_args_list[0][0][0]) + context = MagicMock() + context.args = ['sorted'] + msg_mock.reset_mock() + telegram._whitelist(update=update, context=context) + assert ("Using whitelist `['VolumePairList']` with 4 pairs\n" + "`ETH/BTC, LTC/BTC, NEO/BTC, XRP/BTC`" in msg_mock.call_args_list[0][0][0]) + + context = MagicMock() + context.args = ['baseonly'] + msg_mock.reset_mock() + telegram._whitelist(update=update, context=context) + assert ("Using whitelist `['VolumePairList']` with 4 pairs\n" + "`ETH, LTC, XRP, NEO`" in msg_mock.call_args_list[0][0][0]) + + context = MagicMock() + context.args = ['baseonly', 'sorted'] + msg_mock.reset_mock() + telegram._whitelist(update=update, context=context) + assert ("Using whitelist `['VolumePairList']` with 4 pairs\n" + "`ETH, LTC, NEO, XRP`" in msg_mock.call_args_list[0][0][0]) + def test_blacklist_static(default_conf, update, mocker) -> None: @@ -1507,7 +1575,7 @@ def test_telegram_logs(default_conf, update, mocker) -> None: msg_mock.reset_mock() # Test with changed MaxMessageLength - mocker.patch('freqtrade.rpc.telegram.MAX_TELEGRAM_MESSAGE_LENGTH', 200) + mocker.patch('freqtrade.rpc.telegram.MAX_MESSAGE_LENGTH', 200) context = MagicMock() context.args = [] telegram._logs(update=update, context=context) @@ -1789,7 +1857,6 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en 'leverage': leverage, 'stake_amount': 0.01465333, 'direction': entered, - # 'stake_amount_fiat': 0.0, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'open_rate': 1.099e-05, @@ -1806,6 +1873,33 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en '*Total:* `(0.01465333 BTC, 180.895 USD)`' ) + msg_mock.reset_mock() + telegram.send_msg({ + 'type': message_type, + 'trade_id': 1, + 'enter_tag': enter_signal, + 'exchange': 'Binance', + 'pair': 'ETH/BTC', + 'leverage': leverage, + 'stake_amount': 0.01465333, + 'sub_trade': True, + 'direction': entered, + 'stake_currency': 'BTC', + 'fiat_currency': 'USD', + 'open_rate': 1.099e-05, + 'amount': 1333.3333333333335, + 'open_date': arrow.utcnow().shift(hours=-1) + }) + + assert msg_mock.call_args[0][0] == ( + f'\N{CHECK MARK} *Binance (dry):* {entered}ed ETH/BTC (#1)\n' + f'*Enter Tag:* `{enter_signal}`\n' + '*Amount:* `1333.33333333`\n' + f"{leverage_text}" + '*Open Rate:* `0.00001099`\n' + '*Total:* `(0.01465333 BTC, 180.895 USD)`' + ) + def test_send_msg_sell_notification(default_conf, mocker) -> None: @@ -1840,14 +1934,53 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' - '*Duration:* `1:00:00 (60.0 min)`\n' '*Direction:* `Long`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' - '*Close Rate:* `0.00003201`' + '*Exit Rate:* `0.00003201`\n' + '*Duration:* `1:00:00 (60.0 min)`' ) + msg_mock.reset_mock() + telegram.send_msg({ + 'type': RPCMessageType.EXIT, + 'trade_id': 1, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + 'direction': 'Long', + 'gain': 'loss', + 'limit': 3.201e-05, + 'amount': 1333.3333333333335, + 'order_type': 'market', + 'open_rate': 7.5e-05, + 'current_rate': 3.201e-05, + 'cumulative_profit': -0.15746268, + 'profit_amount': -0.05746268, + 'profit_ratio': -0.57405275, + 'stake_currency': 'ETH', + 'fiat_currency': 'USD', + 'enter_tag': 'buy_signal1', + 'exit_reason': ExitType.STOP_LOSS.value, + 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), + 'close_date': arrow.utcnow(), + 'stake_amount': 0.01, + 'sub_trade': True, + }) + assert msg_mock.call_args[0][0] == ( + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' + '*Unrealized Sub Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' + '*Cumulative Profit:* (`-0.15746268 ETH / -24.812 USD`)\n' + '*Enter Tag:* `buy_signal1`\n' + '*Exit Reason:* `stop_loss`\n' + '*Direction:* `Long`\n' + '*Amount:* `1333.33333333`\n' + '*Open Rate:* `0.00007500`\n' + '*Current Rate:* `0.00003201`\n' + '*Exit Rate:* `0.00003201`\n' + '*Remaining:* `(0.01 ETH, -24.812 USD)`' + ) + msg_mock.reset_mock() telegram.send_msg({ 'type': RPCMessageType.EXIT, @@ -1871,15 +2004,15 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' - '*Unrealized Profit:* `-57.41%`\n' + '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH)`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' - '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n' '*Direction:* `Long`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' - '*Close Rate:* `0.00003201`' + '*Exit Rate:* `0.00003201`\n' + '*Duration:* `1 day, 2:30:00 (1590.0 min)`' ) # Reset singleton function to avoid random breaks telegram._rpc._fiat_converter.convert_amount = old_convamount @@ -1954,15 +2087,15 @@ def test_send_msg_sell_fill_notification(default_conf, mocker, direction, leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exited KEY/ETH (#1)\n' - '*Profit:* `-57.41%`\n' + '*Profit:* `-57.41% (loss: -0.05746268 ETH)`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Exit Reason:* `stop_loss`\n' - '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n' f"*Direction:* `{direction}`\n" f"{leverage_text}" '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' - '*Close Rate:* `0.00003201`' + '*Exit Rate:* `0.00003201`\n' + '*Duration:* `1 day, 2:30:00 (1590.0 min)`' ) @@ -1994,6 +2127,16 @@ def test_startup_notification(default_conf, mocker) -> None: assert msg_mock.call_args[0][0] == '*Custom:* `Hello World`' +def test_send_msg_strategy_msg_notification(default_conf, mocker) -> None: + + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + telegram.send_msg({ + 'type': RPCMessageType.STRATEGY_MSG, + 'msg': 'hello world, Test msg' + }) + assert msg_mock.call_args[0][0] == 'hello world, Test msg' + + def test_send_msg_unknown_type(default_conf, mocker) -> None: telegram, _, _ = get_telegram_testobject(mocker, default_conf) with pytest.raises(NotImplementedError, match=r'Unknown message type: None'): @@ -2080,16 +2223,16 @@ def test_send_msg_sell_notification_no_fiat( leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' - '*Unrealized Profit:* `-57.41%`\n' + '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH)`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Exit Reason:* `stop_loss`\n' - '*Duration:* `2:35:03 (155.1 min)`\n' f'*Direction:* `{direction}`\n' f'{leverage_text}' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' - '*Close Rate:* `0.00003201`' + '*Exit Rate:* `0.00003201`\n' + '*Duration:* `2:35:03 (155.1 min)`' ) diff --git a/tests/strategy/strats/freqai_test_classifier.py b/tests/strategy/strats/freqai_test_classifier.py new file mode 100644 index 000000000..a1e8cb6bf --- /dev/null +++ b/tests/strategy/strats/freqai_test_classifier.py @@ -0,0 +1,138 @@ +import logging +from functools import reduce + +import numpy as np +import pandas as pd +import talib.abstract as ta +from pandas import DataFrame + +from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, merge_informative_pair + + +logger = logging.getLogger(__name__) + + +class freqai_test_classifier(IStrategy): + """ + Test strategy - used for testing freqAI functionalities. + DO not use in production. + """ + + minimal_roi = {"0": 0.1, "240": -1} + + plot_config = { + "main_plot": {}, + "subplots": { + "prediction": {"prediction": {"color": "blue"}}, + "target_roi": { + "target_roi": {"color": "brown"}, + }, + "do_predict": { + "do_predict": {"color": "brown"}, + }, + }, + } + + process_only_new_candles = True + stoploss = -0.05 + use_exit_signal = True + startup_candle_count: int = 300 + can_short = False + + linear_roi_offset = DecimalParameter( + 0.00, 0.02, default=0.005, space="sell", optimize=False, load=True + ) + max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) + + def informative_pairs(self): + whitelist_pairs = self.dp.current_whitelist() + corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] + informative_pairs = [] + for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: + for pair in whitelist_pairs: + informative_pairs.append((pair, tf)) + for pair in corr_pairs: + if pair in whitelist_pairs: + continue # avoid duplication + informative_pairs.append((pair, tf)) + return informative_pairs + + def populate_any_indicators( + self, pair, df, tf, informative=None, set_generalized_indicators=False + ): + + coin = pair.split('/')[0] + + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) + + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + + informative[f"%-{coin}pct-change"] = informative["close"].pct_change() + informative[f"%-{coin}raw_volume"] = informative["volume"] + informative[f"%-{coin}raw_price"] = informative["close"] + + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) + + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) + + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df['&s-up_or_down'] = np.where(df["close"].shift(-100) > df["close"], 'up', 'down') + + return df + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + self.freqai_info = self.config["freqai"] + + dataframe = self.freqai.start(dataframe, metadata, self) + + return dataframe + + def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame: + + enter_long_conditions = [df['&s-up_or_down'] == 'up'] + + if enter_long_conditions: + df.loc[ + reduce(lambda x, y: x & y, enter_long_conditions), ["enter_long", "enter_tag"] + ] = (1, "long") + + enter_short_conditions = [df['&s-up_or_down'] == 'down'] + + if enter_short_conditions: + df.loc[ + reduce(lambda x, y: x & y, enter_short_conditions), ["enter_short", "enter_tag"] + ] = (1, "short") + + return df + + def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame: + + return df diff --git a/tests/strategy/strats/freqai_test_multimodel_strat.py b/tests/strategy/strats/freqai_test_multimodel_strat.py new file mode 100644 index 000000000..cd3327da9 --- /dev/null +++ b/tests/strategy/strats/freqai_test_multimodel_strat.py @@ -0,0 +1,165 @@ +import logging +from functools import reduce + +import pandas as pd +import talib.abstract as ta +from pandas import DataFrame + +from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, merge_informative_pair + + +logger = logging.getLogger(__name__) + + +class freqai_test_multimodel_strat(IStrategy): + """ + Test strategy - used for testing freqAI multimodel functionalities. + DO not use in production. + """ + + minimal_roi = {"0": 0.1, "240": -1} + + plot_config = { + "main_plot": {}, + "subplots": { + "prediction": {"prediction": {"color": "blue"}}, + "target_roi": { + "target_roi": {"color": "brown"}, + }, + "do_predict": { + "do_predict": {"color": "brown"}, + }, + }, + } + + process_only_new_candles = True + stoploss = -0.05 + use_exit_signal = True + startup_candle_count: int = 300 + can_short = False + + linear_roi_offset = DecimalParameter( + 0.00, 0.02, default=0.005, space="sell", optimize=False, load=True + ) + max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) + + def informative_pairs(self): + whitelist_pairs = self.dp.current_whitelist() + corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] + informative_pairs = [] + for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: + for pair in whitelist_pairs: + informative_pairs.append((pair, tf)) + for pair in corr_pairs: + if pair in whitelist_pairs: + continue # avoid duplication + informative_pairs.append((pair, tf)) + return informative_pairs + + def populate_any_indicators( + self, pair, df, tf, informative=None, set_generalized_indicators=False + ): + + coin = pair.split('/')[0] + + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) + + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + + informative[f"%-{coin}pct-change"] = informative["close"].pct_change() + informative[f"%-{coin}raw_volume"] = informative["volume"] + informative[f"%-{coin}raw_price"] = informative["close"] + + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) + + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) + + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) + + df["&-s_range"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .max() + - + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .min() + ) + + return df + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + self.freqai_info = self.config["freqai"] + + dataframe = self.freqai.start(dataframe, metadata, self) + + dataframe["target_roi"] = dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * 1.25 + dataframe["sell_roi"] = dataframe["&-s_close_mean"] - dataframe["&-s_close_std"] * 1.25 + return dataframe + + def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame: + + enter_long_conditions = [df["do_predict"] == 1, df["&-s_close"] > df["target_roi"]] + + if enter_long_conditions: + df.loc[ + reduce(lambda x, y: x & y, enter_long_conditions), ["enter_long", "enter_tag"] + ] = (1, "long") + + enter_short_conditions = [df["do_predict"] == 1, df["&-s_close"] < df["sell_roi"]] + + if enter_short_conditions: + df.loc[ + reduce(lambda x, y: x & y, enter_short_conditions), ["enter_short", "enter_tag"] + ] = (1, "short") + + return df + + def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame: + exit_long_conditions = [df["do_predict"] == 1, df["&-s_close"] < df["sell_roi"] * 0.25] + if exit_long_conditions: + df.loc[reduce(lambda x, y: x & y, exit_long_conditions), "exit_long"] = 1 + + exit_short_conditions = [df["do_predict"] == 1, df["&-s_close"] > df["target_roi"] * 0.25] + if exit_short_conditions: + df.loc[reduce(lambda x, y: x & y, exit_short_conditions), "exit_short"] = 1 + + return df diff --git a/tests/strategy/strats/freqai_test_strat.py b/tests/strategy/strats/freqai_test_strat.py new file mode 100644 index 000000000..792a3952f --- /dev/null +++ b/tests/strategy/strats/freqai_test_strat.py @@ -0,0 +1,153 @@ +import logging +from functools import reduce + +import pandas as pd +import talib.abstract as ta +from pandas import DataFrame + +from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, merge_informative_pair + + +logger = logging.getLogger(__name__) + + +class freqai_test_strat(IStrategy): + """ + Test strategy - used for testing freqAI functionalities. + DO not use in production. + """ + + minimal_roi = {"0": 0.1, "240": -1} + + plot_config = { + "main_plot": {}, + "subplots": { + "prediction": {"prediction": {"color": "blue"}}, + "target_roi": { + "target_roi": {"color": "brown"}, + }, + "do_predict": { + "do_predict": {"color": "brown"}, + }, + }, + } + + process_only_new_candles = True + stoploss = -0.05 + use_exit_signal = True + startup_candle_count: int = 300 + can_short = False + + linear_roi_offset = DecimalParameter( + 0.00, 0.02, default=0.005, space="sell", optimize=False, load=True + ) + max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) + + def informative_pairs(self): + whitelist_pairs = self.dp.current_whitelist() + corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] + informative_pairs = [] + for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: + for pair in whitelist_pairs: + informative_pairs.append((pair, tf)) + for pair in corr_pairs: + if pair in whitelist_pairs: + continue # avoid duplication + informative_pairs.append((pair, tf)) + return informative_pairs + + def populate_any_indicators( + self, pair, df, tf, informative=None, set_generalized_indicators=False + ): + + coin = pair.split('/')[0] + + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) + + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + + informative[f"%-{coin}pct-change"] = informative["close"].pct_change() + informative[f"%-{coin}raw_volume"] = informative["volume"] + informative[f"%-{coin}raw_price"] = informative["close"] + + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) + + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) + + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) + + return df + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + self.freqai_info = self.config["freqai"] + + dataframe = self.freqai.start(dataframe, metadata, self) + + dataframe["target_roi"] = dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * 1.25 + dataframe["sell_roi"] = dataframe["&-s_close_mean"] - dataframe["&-s_close_std"] * 1.25 + return dataframe + + def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame: + + enter_long_conditions = [df["do_predict"] == 1, df["&-s_close"] > df["target_roi"]] + + if enter_long_conditions: + df.loc[ + reduce(lambda x, y: x & y, enter_long_conditions), ["enter_long", "enter_tag"] + ] = (1, "long") + + enter_short_conditions = [df["do_predict"] == 1, df["&-s_close"] < df["sell_roi"]] + + if enter_short_conditions: + df.loc[ + reduce(lambda x, y: x & y, enter_short_conditions), ["enter_short", "enter_tag"] + ] = (1, "short") + + return df + + def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame: + exit_long_conditions = [df["do_predict"] == 1, df["&-s_close"] < df["sell_roi"] * 0.25] + if exit_long_conditions: + df.loc[reduce(lambda x, y: x & y, exit_long_conditions), "exit_long"] = 1 + + exit_short_conditions = [df["do_predict"] == 1, df["&-s_close"] > df["target_roi"] * 0.25] + if exit_short_conditions: + df.loc[reduce(lambda x, y: x & y, exit_short_conditions), "exit_short"] = 1 + + return df diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index 2c7ccbdf2..088ab21d4 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -185,9 +185,12 @@ class StrategyTestV3(IStrategy): return 3.0 - def adjust_trade_position(self, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, - min_stake: Optional[float], max_stake: float, **kwargs): + def adjust_trade_position(self, trade: Trade, current_time: datetime, + current_rate: float, current_profit: float, + min_stake: Optional[float], max_stake: float, + current_entry_rate: float, current_exit_rate: float, + current_entry_profit: float, current_exit_profit: float, + **kwargs) -> Optional[float]: if current_profit < -0.0075: orders = trade.select_filled_orders(trade.entry_side) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index f6996a7a2..65ee05d71 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -12,7 +12,9 @@ from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import load_data from freqtrade.enums import ExitCheckTuple, ExitType, SignalDirection +from freqtrade.enums.hyperoptstate import HyperoptState from freqtrade.exceptions import OperationalException, StrategyError +from freqtrade.optimize.hyperopt_tools import HyperoptStateContainer from freqtrade.optimize.space import SKDecimal from freqtrade.persistence import PairLocks, Trade from freqtrade.resolvers import StrategyResolver @@ -290,6 +292,25 @@ def test_advise_all_indicators(default_conf, testdatadir) -> None: assert len(processed['UNITTEST/BTC']) == 102 # partial candle was removed +def test_populate_any_indicators(default_conf, testdatadir) -> None: + strategy = StrategyResolver.load_strategy(default_conf) + + timerange = TimeRange.parse_timerange('1510694220-1510700340') + data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, + fill_up_missing=True) + processed = strategy.populate_any_indicators('UNITTEST/BTC', data, '5m') + assert processed == data + assert id(processed) == id(data) + assert len(processed['UNITTEST/BTC']) == 102 # partial candle was removed + + +def test_freqai_not_initialized(default_conf) -> None: + strategy = StrategyResolver.load_strategy(default_conf) + strategy.ft_bot_start() + with pytest.raises(OperationalException, match=r'freqAI is not enabled\.'): + strategy.freqai.start() + + def test_advise_all_indicators_copy(mocker, default_conf, testdatadir) -> None: strategy = StrategyResolver.load_strategy(default_conf) aimock = mocker.patch('freqtrade.strategy.interface.IStrategy.advise_indicators') @@ -408,28 +429,31 @@ def test_min_roi_reached3(default_conf, fee) -> None: @pytest.mark.parametrize( - 'profit,adjusted,expected,trailing,custom,profit2,adjusted2,expected2,custom_stop', [ + 'profit,adjusted,expected,liq,trailing,custom,profit2,adjusted2,expected2,custom_stop', [ # Profit, adjusted stoploss(absolute), profit for 2nd call, enable trailing, # enable custom stoploss, expected after 1st call, expected after 2nd call - (0.2, 0.9, ExitType.NONE, False, False, 0.3, 0.9, ExitType.NONE, None), - (0.2, 0.9, ExitType.NONE, False, False, -0.2, 0.9, ExitType.STOP_LOSS, None), - (0.2, 1.14, ExitType.NONE, True, False, 0.05, 1.14, ExitType.TRAILING_STOP_LOSS, None), - (0.01, 0.96, ExitType.NONE, True, False, 0.05, 1, ExitType.NONE, None), - (0.05, 1, ExitType.NONE, True, False, -0.01, 1, ExitType.TRAILING_STOP_LOSS, None), + (0.2, 0.9, ExitType.NONE, None, False, False, 0.3, 0.9, ExitType.NONE, None), + (0.2, 0.9, ExitType.NONE, None, False, False, -0.2, 0.9, ExitType.STOP_LOSS, None), + (0.2, 0.9, ExitType.NONE, 0.8, False, False, -0.2, 0.9, ExitType.LIQUIDATION, None), + (0.2, 1.14, ExitType.NONE, None, True, False, 0.05, 1.14, ExitType.TRAILING_STOP_LOSS, + None), + (0.01, 0.96, ExitType.NONE, None, True, False, 0.05, 1, ExitType.NONE, None), + (0.05, 1, ExitType.NONE, None, True, False, -0.01, 1, ExitType.TRAILING_STOP_LOSS, None), # Default custom case - trails with 10% - (0.05, 0.95, ExitType.NONE, False, True, -0.02, 0.95, ExitType.NONE, None), - (0.05, 0.95, ExitType.NONE, False, True, -0.06, 0.95, ExitType.TRAILING_STOP_LOSS, None), - (0.05, 1, ExitType.NONE, False, True, -0.06, 1, ExitType.TRAILING_STOP_LOSS, + (0.05, 0.95, ExitType.NONE, None, False, True, -0.02, 0.95, ExitType.NONE, None), + (0.05, 0.95, ExitType.NONE, None, False, True, -0.06, 0.95, ExitType.TRAILING_STOP_LOSS, + None), + (0.05, 1, ExitType.NONE, None, False, True, -0.06, 1, ExitType.TRAILING_STOP_LOSS, lambda **kwargs: -0.05), - (0.05, 1, ExitType.NONE, False, True, 0.09, 1.04, ExitType.NONE, + (0.05, 1, ExitType.NONE, None, False, True, 0.09, 1.04, ExitType.NONE, lambda **kwargs: -0.05), - (0.05, 0.95, ExitType.NONE, False, True, 0.09, 0.98, ExitType.NONE, + (0.05, 0.95, ExitType.NONE, None, False, True, 0.09, 0.98, ExitType.NONE, lambda current_profit, **kwargs: -0.1 if current_profit < 0.6 else -(current_profit * 2)), # Error case - static stoploss in place - (0.05, 0.9, ExitType.NONE, False, True, 0.09, 0.9, ExitType.NONE, + (0.05, 0.9, ExitType.NONE, None, False, True, 0.09, 0.9, ExitType.NONE, lambda **kwargs: None), ]) -def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, trailing, custom, +def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, liq, trailing, custom, profit2, adjusted2, expected2, custom_stop) -> None: strategy = StrategyResolver.load_strategy(default_conf) @@ -442,6 +466,7 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili fee_close=fee.return_value, exchange='binance', open_rate=1, + liquidation_price=liq, ) trade.adjust_min_max_rates(trade.open_rate, trade.open_rate) strategy.trailing_stop = trailing @@ -836,7 +861,9 @@ def test_strategy_safe_wrapper_trade_copy(fee): def test_hyperopt_parameters(): + HyperoptStateContainer.set_state(HyperoptState.INDICATORS) from skopt.space import Categorical, Integer, Real + with pytest.raises(OperationalException, match=r"Name is determined.*"): IntParameter(low=0, high=5, default=1, name='hello') @@ -914,6 +941,12 @@ def test_hyperopt_parameters(): assert list(boolpar.range) == [True, False] + HyperoptStateContainer.set_state(HyperoptState.OPTIMIZE) + assert len(list(intpar.range)) == 1 + assert len(list(fltpar.range)) == 1 + assert len(list(catpar.range)) == 1 + assert len(list(boolpar.range)) == 1 + def test_auto_hyperopt_interface(default_conf): default_conf.update({'strategy': 'HyperoptableStrategyV2'}) diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 666ae2b05..b794cdc99 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -34,7 +34,7 @@ def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 6 + assert len(strategies) == 9 assert isinstance(strategies[0], dict) @@ -42,12 +42,16 @@ def test_search_all_strategies_with_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 7 + assert len(strategies) == 10 # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 6 + assert len([x for x in strategies if x['class'] is not None]) == 9 assert len([x for x in strategies if x['class'] is None]) == 1 + directory = Path(__file__).parent / "strats_nonexistingdir" + strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) + assert len(strategies) == 0 + def test_load_strategy(default_conf, result): default_conf.update({'strategy': 'SampleStrategy', diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 66cbd7d9b..138527053 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4,7 +4,6 @@ import logging import time from copy import deepcopy -from math import isclose from typing import List from unittest.mock import ANY, MagicMock, PropertyMock, patch @@ -12,7 +11,7 @@ import arrow import pytest from pandas import DataFrame -from freqtrade.constants import CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT +from freqtrade.constants import CANCEL_REASON, UNLIMITED_STAKE_AMOUNT from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, RPCMessageType, RunMode, SignalDirection, State) from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, @@ -23,9 +22,9 @@ from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections.iprotection import ProtectionReturn from freqtrade.worker import Worker -from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, - log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal, - patch_wallet, patch_whitelist) +from tests.conftest import (create_mock_trades, create_mock_trades_usdt, get_patched_freqtradebot, + get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, + patch_get_signal, patch_wallet, patch_whitelist) from tests.conftest_trades import (MOCK_TRADE_COUNT, entry_side, exit_side, mock_order_1, mock_order_2, mock_order_2_sell, mock_order_3, mock_order_3_sell, mock_order_4, mock_order_5_stoploss, mock_order_6_sell) @@ -68,8 +67,14 @@ def test_process_stopped(mocker, default_conf_usdt) -> None: assert coo_mock.call_count == 1 +def test_process_calls_sendmsg(mocker, default_conf_usdt) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + freqtrade.process() + assert freqtrade.rpc.process_msg_queue.call_count == 1 + + def test_bot_cleanup(mocker, default_conf_usdt, caplog) -> None: - mock_cleanup = mocker.patch('freqtrade.freqtradebot.cleanup_db') + mock_cleanup = mocker.patch('freqtrade.freqtradebot.Trade.commit') coo_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cancel_all_open_orders') freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) freqtrade.cleanup() @@ -468,8 +473,6 @@ def test_create_trade_no_signal(default_conf_usdt, fee, mocker) -> None: freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade, enter_long=False, exit_long=False) - Trade.query = MagicMock() - Trade.query.filter = MagicMock() assert not freqtrade.create_trade('ETH/USDT') @@ -563,7 +566,7 @@ def test_process_trade_creation(default_conf_usdt, ticker_usdt, limit_order, lim assert trade.open_date is not None assert trade.exchange == 'binance' assert trade.open_rate == ticker_usdt.return_value[ticker_side] - assert isclose(trade.amount, 60 / ticker_usdt.return_value[ticker_side]) + assert pytest.approx(trade.amount) == 60 / ticker_usdt.return_value[ticker_side] assert log_has( f'{"Short" if is_short else "Long"} signal found: about create a new trade for ETH/USDT ' @@ -672,6 +675,7 @@ def test_process_trade_no_whitelist_pair(default_conf_usdt, ticker_usdt, limit_b open_rate=0.001, exchange='binance', )) + Trade.commit() assert pair not in freqtrade.active_pair_whitelist freqtrade.process() @@ -837,8 +841,8 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, # In case of closed order order['status'] = 'closed' - order['price'] = 10 - order['cost'] = 100 + order['average'] = 10 + order['cost'] = 300 order['id'] = '444' mocker.patch('freqtrade.exchange.Exchange.create_order', @@ -849,7 +853,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, assert trade assert trade.open_order_id is None assert trade.open_rate == 10 - assert trade.stake_amount == round(order['price'] * order['filled'] / leverage, 8) + assert trade.stake_amount == round(order['average'] * order['filled'] / leverage, 8) assert pytest.approx(trade.liquidation_price) == liq_price # In case of rejected or expired order and partially filled @@ -857,8 +861,8 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, order['amount'] = 30.0 order['filled'] = 20.0 order['remaining'] = 10.00 - order['price'] = 0.5 - order['cost'] = 15.0 + order['average'] = 0.5 + order['cost'] = 10.0 order['id'] = '555' mocker.patch('freqtrade.exchange.Exchange.create_order', MagicMock(return_value=order)) @@ -866,9 +870,9 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade = Trade.query.all()[3] trade.is_short = is_short assert trade - assert trade.open_order_id == '555' + assert trade.open_order_id is None assert trade.open_rate == 0.5 - assert trade.stake_amount == round(order['price'] * order['filled'] / leverage, 8) + assert trade.stake_amount == round(order['average'] * order['filled'] / leverage, 8) # Test with custom stake order['status'] = 'open' @@ -895,7 +899,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, order['amount'] = 30.0 * leverage order['filled'] = 0.0 order['remaining'] = 30.0 - order['price'] = 0.5 + order['average'] = 0.5 order['cost'] = 0.0 order['id'] = '66' mocker.patch('freqtrade.exchange.Exchange.create_order', @@ -967,6 +971,14 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade.is_short = is_short assert pytest.approx(trade.stake_amount) == 500 + order['id'] = '55673' + + freqtrade.strategy.leverage.reset_mock() + assert freqtrade.execute_entry(pair, 200, leverage_=3) + assert freqtrade.strategy.leverage.call_count == 0 + trade = Trade.query.all()[10] + assert trade.leverage == 1 if trading_mode == 'spot' else 3 + @pytest.mark.parametrize("is_short", [False, True]) def test_execute_entry_confirm_error(mocker, default_conf_usdt, fee, limit_order, is_short) -> None: @@ -1077,7 +1089,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ 'last': 1.9 }), create_order=MagicMock(side_effect=[ - {'id': enter_order['id']}, + enter_order, exit_order, ]), get_fee=fee, @@ -1103,20 +1115,20 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ # should do nothing and return false trade.is_open = True trade.open_order_id = None - trade.stoploss_order_id = 100 + trade.stoploss_order_id = "100" hanging_stoploss_order = MagicMock(return_value={'status': 'open'}) mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', hanging_stoploss_order) assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert trade.stoploss_order_id == 100 + assert trade.stoploss_order_id == "100" # Third case: when stoploss was set but it was canceled for some reason # should set a stoploss immediately and return False caplog.clear() trade.is_open = True trade.open_order_id = None - trade.stoploss_order_id = 100 + trade.stoploss_order_id = "100" canceled_stoploss_order = MagicMock(return_value={'status': 'canceled'}) mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', canceled_stoploss_order) @@ -1787,7 +1799,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, # stoploss initially at 20% as edge dictated it. assert freqtrade.handle_trade(trade) is False assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert isclose(trade.stop_loss, 1.76) + assert pytest.approx(trade.stop_loss) == 1.76 cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock() @@ -1804,7 +1816,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, assert freqtrade.handle_stoploss_on_exchange(trade) is False # stoploss should remain the same - assert isclose(trade.stop_loss, 1.76) + assert pytest.approx(trade.stop_loss) == 1.76 # stoploss on exchange should not be canceled cancel_order_mock.assert_not_called() @@ -2033,6 +2045,7 @@ def test_update_trade_state_exception(mocker, default_conf_usdt, is_short, limit trade = MagicMock() trade.open_order_id = '123' + trade.amount = 123 # Test raise of OperationalException exception mocker.patch( @@ -2157,7 +2170,7 @@ def test_handle_trade( assert trade.close_rate == (2.0 if is_short else 2.2) assert pytest.approx(trade.close_profit) == close_profit - assert trade.calc_profit(trade.close_rate) == 5.685 + assert pytest.approx(trade.calc_profit(trade.close_rate)) == 5.685 assert trade.close_date is not None assert trade.exit_reason == 'sell_signal1' @@ -2346,9 +2359,9 @@ def test_close_trade( trade.is_short = is_short assert trade - oobj = Order.parse_from_ccxt_object(enter_order, enter_order['symbol'], 'buy') + oobj = Order.parse_from_ccxt_object(enter_order, enter_order['symbol'], trade.enter_side) trade.update_trade(oobj) - oobj = Order.parse_from_ccxt_object(exit_order, exit_order['symbol'], 'sell') + oobj = Order.parse_from_ccxt_object(exit_order, exit_order['symbol'], trade.exit_side) trade.update_trade(oobj) assert trade.is_open is False @@ -2391,8 +2404,8 @@ def test_manage_open_orders_entry_usercustom( 'freqtrade.exchange.Exchange', fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=old_order), - cancel_order_with_result=cancel_order_wr_mock, cancel_order=cancel_order_mock, + cancel_order_with_result=cancel_order_wr_mock, get_fee=fee ) freqtrade = FreqtradeBot(default_conf_usdt) @@ -2400,6 +2413,7 @@ def test_manage_open_orders_entry_usercustom( open_trade.orders[0].side = 'sell' if is_short else 'buy' open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' Trade.query.session.add(open_trade) + Trade.commit() # Ensure default is to return empty (so not mocked yet) freqtrade.manage_open_orders() @@ -2440,7 +2454,9 @@ def test_manage_open_orders_entry( ) -> None: old_order = limit_sell_order_old if is_short else limit_buy_order_old rpc_mock = patch_RPCManager(mocker) - old_order['id'] = open_trade.open_order_id + open_trade.open_order_id = old_order['id'] + order = Order.parse_from_ccxt_object(old_order, 'mocked', 'buy') + open_trade.orders[0] = order limit_buy_cancel = deepcopy(old_order) limit_buy_cancel['status'] = 'canceled' cancel_order_mock = MagicMock(return_value=limit_buy_cancel) @@ -2456,6 +2472,7 @@ def test_manage_open_orders_entry( open_trade.is_short = is_short Trade.query.session.add(open_trade) + Trade.commit() freqtrade.strategy.check_entry_timeout = MagicMock(return_value=False) freqtrade.strategy.adjust_entry_price = MagicMock(return_value=1234) @@ -2493,6 +2510,7 @@ def test_adjust_entry_cancel( open_trade.is_short = is_short Trade.query.session.add(open_trade) + Trade.commit() # Timeout to not interfere freqtrade.strategy.ft_check_timed_out = MagicMock(return_value=False) @@ -2533,6 +2551,7 @@ def test_adjust_entry_maintain_replace( open_trade.is_short = is_short Trade.query.session.add(open_trade) + Trade.commit() # Timeout to not interfere freqtrade.strategy.ft_check_timed_out = MagicMock(return_value=False) @@ -2585,6 +2604,7 @@ def test_check_handle_cancelled_buy( open_trade.orders = [] open_trade.is_short = is_short Trade.query.session.add(open_trade) + Trade.commit() # check it does cancel buy orders over the time limit freqtrade.manage_open_orders() @@ -2615,6 +2635,7 @@ def test_manage_open_orders_buy_exception( open_trade.is_short = is_short Trade.query.session.add(open_trade) + Trade.commit() # check it does cancel buy orders over the time limit freqtrade.manage_open_orders() @@ -2631,7 +2652,9 @@ def test_manage_open_orders_exit_usercustom( is_short, open_trade_usdt, caplog ) -> None: default_conf_usdt["unfilledtimeout"] = {"entry": 1440, "exit": 1440, "exit_timeout_count": 1} - limit_sell_order_old['id'] = open_trade_usdt.open_order_id + open_trade_usdt.open_order_id = limit_sell_order_old['id'] + order = Order.parse_from_ccxt_object(limit_sell_order_old, 'mocked', 'sell') + open_trade_usdt.orders[0] = order if is_short: limit_sell_order_old['side'] = 'buy' open_trade_usdt.is_short = is_short @@ -2654,6 +2677,7 @@ def test_manage_open_orders_exit_usercustom( open_trade_usdt.is_open = False Trade.query.session.add(open_trade_usdt) + Trade.commit() # Ensure default is false freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 @@ -2736,6 +2760,7 @@ def test_manage_open_orders_exit( open_trade_usdt.is_short = is_short Trade.query.session.add(open_trade_usdt) + Trade.commit() freqtrade.strategy.check_exit_timeout = MagicMock(return_value=False) freqtrade.strategy.check_entry_timeout = MagicMock(return_value=False) @@ -2776,6 +2801,7 @@ def test_check_handle_cancelled_exit( open_trade_usdt.is_short = is_short Trade.query.session.add(open_trade_usdt) + Trade.commit() # check it does cancel sell orders over the time limit freqtrade.manage_open_orders() @@ -2812,6 +2838,7 @@ def test_manage_open_orders_partial( freqtrade = FreqtradeBot(default_conf_usdt) prior_stake = open_trade.stake_amount Trade.query.session.add(open_trade) + Trade.commit() # check it does cancel buy orders over the time limit # note this is for a partially-complete buy order @@ -2856,6 +2883,7 @@ def test_manage_open_orders_partial_fee( open_trade.fee_open = fee() open_trade.fee_close = fee() Trade.query.session.add(open_trade) + Trade.commit() # cancelling a half-filled order should update the amount to the bought amount # and apply fees if necessary. freqtrade.manage_open_orders() @@ -2905,6 +2933,7 @@ def test_manage_open_orders_partial_except( open_trade.fee_open = fee() open_trade.fee_close = fee() Trade.query.session.add(open_trade) + Trade.commit() # cancelling a half-filled order should update the amount to the bought amount # and apply fees if necessary. freqtrade.manage_open_orders() @@ -2943,6 +2972,7 @@ def test_manage_open_orders_exception(default_conf_usdt, ticker_usdt, open_trade freqtrade = FreqtradeBot(default_conf_usdt) Trade.query.session.add(open_trade_usdt) + Trade.commit() caplog.clear() freqtrade.manage_open_orders() @@ -3244,6 +3274,9 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_ 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY, + 'sub_trade': False, + 'cumulative_profit': 0.0, + 'stake_amount': pytest.approx(60), } == last_msg @@ -3304,6 +3337,9 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY, + 'sub_trade': False, + 'cumulative_profit': 0.0, + 'stake_amount': pytest.approx(60), } == last_msg @@ -3385,6 +3421,9 @@ def test_execute_trade_exit_custom_exit_price( 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY, + 'sub_trade': False, + 'cumulative_profit': 0.0, + 'stake_amount': pytest.approx(60), } == last_msg @@ -3423,7 +3462,7 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run( trade.stop_loss = 2.0 * 1.01 if is_short else 2.0 * 0.99 freqtrade.execute_trade_exit( - trade=trade, limit=(ticker_usdt_sell_up if is_short else ticker_usdt_sell_down())['bid'], + trade=trade, limit=trade.stop_loss, exit_check=ExitCheckTuple(exit_type=ExitType.STOP_LOSS)) assert rpc_mock.call_count == 2 @@ -3453,6 +3492,9 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run( 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY, + 'sub_trade': False, + 'cumulative_profit': 0.0, + 'stake_amount': pytest.approx(60), } == last_msg @@ -3684,7 +3726,7 @@ def test_execute_trade_exit_market_order( ) assert not trade.is_open - assert trade.close_profit == profit_ratio + assert pytest.approx(trade.close_profit) == profit_ratio assert rpc_mock.call_count == 4 last_msg = rpc_mock.call_args_list[-2][0][0] @@ -3712,6 +3754,9 @@ def test_execute_trade_exit_market_order( 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY, + 'sub_trade': False, + 'cumulative_profit': 0.0, + 'stake_amount': pytest.approx(60), } == last_msg @@ -3783,7 +3828,7 @@ def test_exit_profit_only( 'last': bid }), create_order=MagicMock(side_effect=[ - limit_order_open[eside], + limit_order[eside], {'id': 1234553382}, ]), get_fee=fee, @@ -4075,7 +4120,7 @@ def test_trailing_stop_loss_positive( 'last': enter_price - (-0.01 if is_short else 0.01), }), create_order=MagicMock(side_effect=[ - limit_order_open[eside], + limit_order[eside], {'id': 1234553382}, ]), get_fee=fee, @@ -4110,6 +4155,7 @@ def test_trailing_stop_loss_positive( 'last': enter_price + (-0.06 if is_short else 0.06), }) ) + caplog.clear() # stop-loss not reached, adjusted stoploss assert freqtrade.handle_trade(trade) is False caplog_text = (f"ETH/USDT - Using positive stoploss: 0.01 offset: {offset} profit: " @@ -4490,11 +4536,8 @@ def test_get_real_amount_wrong_amount_rounding(default_conf_usdt, trades_for_ord order_obj = Order.parse_from_ccxt_object(buy_order_fee, 'LTC/ETH', 'buy') # Amount changes by fee amount. - assert isclose( - freqtrade.get_real_amount(trade, limit_buy_order_usdt, order_obj), - amount - (amount * 0.001), - abs_tol=MATH_CLOSE_PREC, - ) + assert pytest.approx(freqtrade.get_real_amount( + trade, limit_buy_order_usdt, order_obj)) == amount - (amount * 0.001) def test_get_real_amount_open_trade_usdt(default_conf_usdt, fee, mocker): @@ -4519,6 +4562,76 @@ def test_get_real_amount_open_trade_usdt(default_conf_usdt, fee, mocker): assert freqtrade.get_real_amount(trade, order, order_obj) == amount +def test_get_real_amount_in_point(default_conf_usdt, buy_order_fee, fee, mocker, caplog): + limit_buy_order_usdt = deepcopy(buy_order_fee) + + # Fees amount in "POINT" + trades = [{ + "info": { + }, + "id": "some_trade_id", + "timestamp": 1660092505903, + "datetime": "2022-08-10T00:48:25.903Z", + "symbol": "CEL/USDT", + "order": "some_order_id", + "type": None, + "side": "sell", + "takerOrMaker": "taker", + "price": 1.83255, + "amount": 83.126, + "cost": 152.3325513, + "fee": { + "currency": "POINT", + "cost": 0.3046651026 + }, + "fees": [ + { + "cost": "0", + "currency": "USDT" + }, + { + "cost": "0", + "currency": "GT" + }, + { + "cost": "0.3046651026", + "currency": "POINT" + } + ] + }] + + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades) + amount = float(sum(x['amount'] for x in trades)) + trade = Trade( + pair='CEL/USDT', + amount=amount, + exchange='binance', + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.245441, + open_order_id="123456" + ) + limit_buy_order_usdt['amount'] = amount + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + + order_obj = Order.parse_from_ccxt_object(buy_order_fee, 'LTC/ETH', 'buy') + res = freqtrade.get_real_amount(trade, limit_buy_order_usdt, order_obj) + assert res == amount + assert trade.fee_open_currency is None + assert trade.fee_open_cost is None + message = "Not updating buy-fee - rate: None, POINT." + assert log_has(message, caplog) + caplog.clear() + freqtrade.config['exchange']['unknown_fee_rate'] = 1 + res = freqtrade.get_real_amount(trade, limit_buy_order_usdt, order_obj) + assert res == amount + assert trade.fee_open_currency == 'POINT' + assert pytest.approx(trade.fee_open_cost) == 0.3046651026 + assert trade.fee_open == 0.002 + assert trade.fee_open != fee.return_value + assert not log_has(message, caplog) + + @pytest.mark.parametrize('amount,fee_abs,wallet,amount_exp', [ (8.0, 0.0, 10, 8), (8.0, 0.0, 0, 8), @@ -4626,7 +4739,7 @@ def test_order_book_entry_pricing1(mocker, default_conf_usdt, order_book_l2, exc with pytest.raises(PricingError): freqtrade.exchange.get_rate('ETH/USDT', side="entry", is_short=False, refresh=True) assert log_has_re( - r'Entry Price at location 1 from orderbook could not be determined.', caplog) + r'ETH/USDT - Entry Price at location 1 from orderbook could not be determined.', caplog) else: assert freqtrade.exchange.get_rate( 'ETH/USDT', side="entry", is_short=False, refresh=True) == 0.043935 @@ -4705,8 +4818,9 @@ def test_order_book_exit_pricing( return_value={'bids': [[]], 'asks': [[]]}) with pytest.raises(PricingError): freqtrade.handle_trade(trade) - assert log_has_re(r'Exit Price at location 1 from orderbook could not be determined\..*', - caplog) + assert log_has_re( + r"ETH/USDT - Exit Price at location 1 from orderbook could not be determined\..*", + caplog) def test_startup_state(default_conf_usdt, mocker): @@ -4853,6 +4967,31 @@ def test_startup_update_open_orders(mocker, default_conf_usdt, fee, caplog, is_s assert hto_mock.call_args_list[1][0][0]['status'] == 'canceled' +@pytest.mark.usefixtures("init_persistence") +def test_startup_backpopulate_precision(mocker, default_conf_usdt, fee, caplog): + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + create_mock_trades_usdt(fee) + + trades = Trade.get_trades().all() + trades[-1].exchange = 'some_other_exchange' + for trade in trades: + assert trade.price_precision is None + assert trade.amount_precision is None + assert trade.precision_mode is None + + freqtrade.startup_backpopulate_precision() + trades = Trade.get_trades().all() + for trade in trades: + if trade.exchange == 'some_other_exchange': + assert trade.price_precision is None + assert trade.amount_precision is None + assert trade.precision_mode is None + else: + assert trade.price_precision is not None + assert trade.amount_precision is not None + assert trade.precision_mode is not None + + @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize("is_short", [False, True]) def test_update_closed_trades_without_assigned_fees(mocker, default_conf_usdt, fee, is_short): @@ -5379,7 +5518,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: 'status': None, 'price': 9, 'amount': 12, - 'cost': 100, + 'cost': 108, 'ft_is_open': True, 'id': '651', 'order_id': '651' @@ -5474,7 +5613,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: assert trade.open_order_id is None assert pytest.approx(trade.open_rate) == 9.90909090909 assert trade.amount == 22 - assert trade.stake_amount == 218 + assert pytest.approx(trade.stake_amount) == 218 orders = Order.query.all() assert orders @@ -5527,6 +5666,329 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: # Make sure the closed order is found as the second order. order = trade.select_order('buy', False) assert order.order_id == '652' + closed_sell_dca_order_1 = { + 'ft_pair': pair, + 'status': 'closed', + 'ft_order_side': 'sell', + 'side': 'sell', + 'type': 'limit', + 'price': 8, + 'average': 8, + 'amount': 15, + 'filled': 15, + 'cost': 120, + 'ft_is_open': False, + 'id': '653', + 'order_id': '653' + } + mocker.patch('freqtrade.exchange.Exchange.create_order', + MagicMock(return_value=closed_sell_dca_order_1)) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', + MagicMock(return_value=closed_sell_dca_order_1)) + mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + MagicMock(return_value=closed_sell_dca_order_1)) + assert freqtrade.execute_trade_exit(trade=trade, limit=8, + exit_check=ExitCheckTuple(exit_type=ExitType.PARTIAL_EXIT), + sub_trade_amt=15) + + # Assert trade is as expected (averaged dca) + trade = Trade.query.first() + assert trade + assert trade.open_order_id is None + assert trade.is_open + assert trade.amount == 22 + assert trade.stake_amount == 192.05405405405406 + assert pytest.approx(trade.open_rate) == 8.729729729729 + + orders = Order.query.all() + assert orders + assert len(orders) == 4 + + # Make sure the closed order is found as the second order. + order = trade.select_order('sell', False) + assert order.order_id == '653' + + +def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: + """ + TODO: Should be adjusted to test both long and short + buy 100 @ 11 + sell 50 @ 8 + sell 50 @ 16 + """ + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_wallet(mocker, free=10000) + default_conf_usdt.update({ + "position_adjustment_enable": True, + "dry_run": False, + "stake_amount": 200.0, + "dry_run_wallet": 1000.0, + }) + freqtrade = FreqtradeBot(default_conf_usdt) + freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True) + bid = 11 + amount = 100 + buy_rate_mock = MagicMock(return_value=bid) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_rate=buy_rate_mock, + fetch_ticker=MagicMock(return_value={ + 'bid': 10, + 'ask': 12, + 'last': 11 + }), + get_min_pair_stake_amount=MagicMock(return_value=1), + get_fee=fee, + ) + pair = 'ETH/USDT' + # Initial buy + closed_successful_buy_order = { + 'pair': pair, + 'ft_pair': pair, + 'ft_order_side': 'buy', + 'side': 'buy', + 'type': 'limit', + 'status': 'closed', + 'price': bid, + 'average': bid, + 'cost': bid * amount, + 'amount': amount, + 'filled': amount, + 'ft_is_open': False, + 'id': '600', + 'order_id': '600' + } + mocker.patch('freqtrade.exchange.Exchange.create_order', + MagicMock(return_value=closed_successful_buy_order)) + mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + MagicMock(return_value=closed_successful_buy_order)) + assert freqtrade.execute_entry(pair, amount) + # Should create an closed trade with an no open order id + # Order is filled and trade is open + orders = Order.query.all() + assert orders + assert len(orders) == 1 + trade = Trade.query.first() + assert trade + assert trade.is_open is True + assert trade.open_order_id is None + assert trade.open_rate == bid + assert trade.stake_amount == bid * amount + + # Assume it does nothing since order is closed and trade is open + freqtrade.update_closed_trades_without_assigned_fees() + + trade = Trade.query.first() + assert trade + assert trade.is_open is True + assert trade.open_order_id is None + assert trade.open_rate == bid + assert trade.stake_amount == bid * amount + assert not trade.fee_updated(trade.entry_side) + + freqtrade.manage_open_orders() + + trade = Trade.query.first() + assert trade + assert trade.is_open is True + assert trade.open_order_id is None + assert trade.open_rate == bid + assert trade.stake_amount == bid * amount + assert not trade.fee_updated(trade.entry_side) + + amount = 50 + ask = 8 + closed_sell_dca_order_1 = { + 'ft_pair': pair, + 'status': 'closed', + 'ft_order_side': 'sell', + 'side': 'sell', + 'type': 'limit', + 'price': ask, + 'average': ask, + 'amount': amount, + 'filled': amount, + 'cost': amount * ask, + 'ft_is_open': False, + 'id': '601', + 'order_id': '601' + } + mocker.patch('freqtrade.exchange.Exchange.create_order', + MagicMock(return_value=closed_sell_dca_order_1)) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', + MagicMock(return_value=closed_sell_dca_order_1)) + mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + MagicMock(return_value=closed_sell_dca_order_1)) + assert freqtrade.execute_trade_exit(trade=trade, limit=ask, + exit_check=ExitCheckTuple(exit_type=ExitType.PARTIAL_EXIT), + sub_trade_amt=amount) + trades: List[Trade] = trade.get_open_trades_without_assigned_fees() + assert len(trades) == 1 + # Assert trade is as expected (averaged dca) + + trade = Trade.query.first() + assert trade + assert trade.open_order_id is None + assert trade.amount == 50 + assert trade.open_rate == 11 + assert trade.stake_amount == 550 + assert pytest.approx(trade.realized_profit) == -152.375 + assert pytest.approx(trade.close_profit_abs) == -152.375 + + orders = Order.query.all() + assert orders + assert len(orders) == 2 + # Make sure the closed order is found as the second order. + order = trade.select_order('sell', False) + assert order.order_id == '601' + + amount = 50 + ask = 16 + closed_sell_dca_order_2 = { + 'ft_pair': pair, + 'status': 'closed', + 'ft_order_side': 'sell', + 'side': 'sell', + 'type': 'limit', + 'price': ask, + 'average': ask, + 'amount': amount, + 'filled': amount, + 'cost': amount * ask, + 'ft_is_open': False, + 'id': '602', + 'order_id': '602' + } + mocker.patch('freqtrade.exchange.Exchange.create_order', + MagicMock(return_value=closed_sell_dca_order_2)) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', + MagicMock(return_value=closed_sell_dca_order_2)) + mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + MagicMock(return_value=closed_sell_dca_order_2)) + assert freqtrade.execute_trade_exit(trade=trade, limit=ask, + exit_check=ExitCheckTuple(exit_type=ExitType.PARTIAL_EXIT), + sub_trade_amt=amount) + # Assert trade is as expected (averaged dca) + + trade = Trade.query.first() + assert trade + assert trade.open_order_id is None + assert trade.amount == 50 + assert trade.open_rate == 11 + assert trade.stake_amount == 550 + # Trade fully realized + assert pytest.approx(trade.realized_profit) == 94.25 + assert pytest.approx(trade.close_profit_abs) == 94.25 + orders = Order.query.all() + assert orders + assert len(orders) == 3 + + # Make sure the closed order is found as the second order. + order = trade.select_order('sell', False) + assert order.order_id == '602' + assert trade.is_open is False + + +@pytest.mark.parametrize('data', [ + ( + # tuple 1 - side amount, price + # tuple 2 - amount, open_rate, stake_amount, cumulative_profit, realized_profit, rel_profit + (('buy', 100, 10), (100.0, 10.0, 1000.0, 0.0, None, None)), + (('buy', 100, 15), (200.0, 12.5, 2500.0, 0.0, None, None)), + (('sell', 50, 12), (150.0, 12.5, 1875.0, -28.0625, -28.0625, -0.044788)), + (('sell', 100, 20), (50.0, 12.5, 625.0, 713.8125, 741.875, 0.59201995)), + (('sell', 50, 5), (50.0, 12.5, 625.0, 336.625, 336.625, 0.1343142)), # final profit (sum) + ), + ( + (('buy', 100, 3), (100.0, 3.0, 300.0, 0.0, None, None)), + (('buy', 100, 7), (200.0, 5.0, 1000.0, 0.0, None, None)), + (('sell', 100, 11), (100.0, 5.0, 500.0, 596.0, 596.0, 1.189027)), + (('buy', 150, 15), (250.0, 11.0, 2750.0, 596.0, 596.0, 1.189027)), + (('sell', 100, 19), (150.0, 11.0, 1650.0, 1388.5, 792.5, 0.7186579)), + (('sell', 150, 23), (150.0, 11.0, 1650.0, 3175.75, 3175.75, 0.9747170)), # final profit + ) +]) +def test_position_adjust3(mocker, default_conf_usdt, fee, data) -> None: + default_conf_usdt.update({ + "position_adjustment_enable": True, + "dry_run": False, + "stake_amount": 200.0, + "dry_run_wallet": 1000.0, + }) + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_wallet(mocker, free=10000) + freqtrade = FreqtradeBot(default_conf_usdt) + trade = None + freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True) + for idx, (order, result) in enumerate(data): + amount = order[1] + price = order[2] + price_mock = MagicMock(return_value=price) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_rate=price_mock, + fetch_ticker=MagicMock(return_value={ + 'bid': 10, + 'ask': 12, + 'last': 11 + }), + get_min_pair_stake_amount=MagicMock(return_value=1), + get_fee=fee, + ) + pair = 'ETH/USDT' + closed_successful_order = { + 'pair': pair, + 'ft_pair': pair, + 'ft_order_side': order[0], + 'side': order[0], + 'type': 'limit', + 'status': 'closed', + 'price': price, + 'average': price, + 'cost': price * amount, + 'amount': amount, + 'filled': amount, + 'ft_is_open': False, + 'id': f'60{idx}', + 'order_id': f'60{idx}' + } + mocker.patch('freqtrade.exchange.Exchange.create_order', + MagicMock(return_value=closed_successful_order)) + mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + MagicMock(return_value=closed_successful_order)) + if order[0] == 'buy': + assert freqtrade.execute_entry(pair, amount, trade=trade) + else: + assert freqtrade.execute_trade_exit( + trade=trade, limit=price, + exit_check=ExitCheckTuple(exit_type=ExitType.PARTIAL_EXIT), + sub_trade_amt=amount) + + orders1 = Order.query.all() + assert orders1 + assert len(orders1) == idx + 1 + + trade = Trade.query.first() + assert trade + if idx < len(data) - 1: + assert trade.is_open is True + assert trade.open_order_id is None + assert trade.amount == result[0] + assert trade.open_rate == result[1] + assert trade.stake_amount == result[2] + assert pytest.approx(trade.realized_profit) == result[3] + assert pytest.approx(trade.close_profit_abs) == result[4] + assert pytest.approx(trade.close_profit) == result[5] + + order_obj = trade.select_order(order[0], False) + assert order_obj.order_id == f'60{idx}' + + trade = Trade.query.first() + assert trade + assert trade.open_order_id is None + assert trade.is_open is False def test_process_open_trade_positions_exception(mocker, default_conf_usdt, fee, caplog) -> None: @@ -5550,9 +6012,25 @@ def test_check_and_call_adjust_trade_position(mocker, default_conf_usdt, fee, ca "max_entry_position_adjustment": 0, }) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) - + buy_rate_mock = MagicMock(return_value=10) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_rate=buy_rate_mock, + fetch_ticker=MagicMock(return_value={ + 'bid': 10, + 'ask': 12, + 'last': 11 + }), + get_min_pair_stake_amount=MagicMock(return_value=1), + get_fee=fee, + ) create_mock_trades(fee) caplog.set_level(logging.DEBUG) - + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=10) freqtrade.process_open_trade_positions() assert log_has_re(r"Max adjustment entries for .* has been reached\.", caplog) + + caplog.clear() + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-10) + freqtrade.process_open_trade_positions() + assert log_has_re(r"LIMIT_SELL has been fulfilled.*", caplog) diff --git a/tests/test_integration.py b/tests/test_integration.py index 83f54becb..dd3488f81 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -6,7 +6,7 @@ from freqtrade.enums import ExitCheckTuple, ExitType from freqtrade.persistence import Trade from freqtrade.persistence.models import Order from freqtrade.rpc.rpc import RPC -from tests.conftest import get_patched_freqtradebot, patch_get_signal +from tests.conftest import get_patched_freqtradebot, log_has_re, patch_get_signal def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, @@ -189,7 +189,7 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, mocker, balance_rati assert len(trades) == 5 for trade in trades: - assert trade.stake_amount == result1 + assert pytest.approx(trade.stake_amount) == result1 # Reset trade open order id's trade.open_order_id = None trades = Trade.get_open_trades() @@ -220,8 +220,6 @@ def test_dca_buying(default_conf_usdt, ticker_usdt, fee, mocker) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker_usdt, get_fee=fee, - amount_to_precision=lambda s, x, y: y, - price_to_precision=lambda s, x, y: y, ) patch_get_signal(freqtrade) @@ -249,7 +247,7 @@ def test_dca_buying(default_conf_usdt, ticker_usdt, fee, mocker) -> None: assert len(trade.orders) == 2 for o in trade.orders: assert o.status == "closed" - assert trade.stake_amount == 120 + assert pytest.approx(trade.stake_amount) == 120 # Open-rate averaged between 2.0 and 2.0 * 0.995 assert trade.open_rate < 2.0 @@ -259,11 +257,11 @@ def test_dca_buying(default_conf_usdt, ticker_usdt, fee, mocker) -> None: freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 2 - assert trade.stake_amount == 120 + assert pytest.approx(trade.stake_amount) == 120 assert trade.orders[0].amount == 30 - assert trade.orders[1].amount == 60 / ticker_usdt_modif['bid'] + assert pytest.approx(trade.orders[1].amount) == 60 / ticker_usdt_modif['bid'] - assert trade.amount == trade.orders[0].amount + trade.orders[1].amount + assert pytest.approx(trade.amount) == trade.orders[0].amount + trade.orders[1].amount assert trade.nr_of_successful_buys == 2 assert trade.nr_of_successful_entries == 2 @@ -274,7 +272,7 @@ def test_dca_buying(default_conf_usdt, ticker_usdt, fee, mocker) -> None: assert trade.is_open is False assert trade.orders[0].amount == 30 assert trade.orders[0].side == 'buy' - assert trade.orders[1].amount == 60 / ticker_usdt_modif['bid'] + assert pytest.approx(trade.orders[1].amount) == 60 / ticker_usdt_modif['bid'] # Sold everything assert trade.orders[-1].side == 'sell' assert trade.orders[2].amount == trade.amount @@ -291,7 +289,7 @@ def test_dca_short(default_conf_usdt, ticker_usdt, fee, mocker) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker_usdt, get_fee=fee, - amount_to_precision=lambda s, x, y: y, + amount_to_precision=lambda s, x, y: round(y, 4), price_to_precision=lambda s, x, y: y, ) @@ -303,6 +301,7 @@ def test_dca_short(default_conf_usdt, ticker_usdt, fee, mocker) -> None: assert len(trade.orders) == 1 assert pytest.approx(trade.stake_amount) == 60 assert trade.open_rate == 2.02 + assert trade.orders[0].amount == trade.amount # No adjustment freqtrade.process() trade = Trade.get_trades().first() @@ -331,8 +330,7 @@ def test_dca_short(default_conf_usdt, ticker_usdt, fee, mocker) -> None: trade = Trade.get_trades().first() assert len(trade.orders) == 2 assert pytest.approx(trade.stake_amount) == 120 - # assert trade.orders[0].amount == 30 - assert trade.orders[1].amount == 60 / ticker_usdt_modif['ask'] + assert trade.orders[1].amount == round(60 / ticker_usdt_modif['ask'], 4) assert trade.amount == trade.orders[0].amount + trade.orders[1].amount assert trade.nr_of_successful_entries == 2 @@ -344,7 +342,7 @@ def test_dca_short(default_conf_usdt, ticker_usdt, fee, mocker) -> None: assert trade.is_open is False # assert trade.orders[0].amount == 30 assert trade.orders[0].side == 'sell' - assert trade.orders[1].amount == 60 / ticker_usdt_modif['ask'] + assert trade.orders[1].amount == round(60 / ticker_usdt_modif['ask'], 4) # Sold everything assert trade.orders[-1].side == 'buy' assert trade.orders[2].amount == trade.amount @@ -455,3 +453,60 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, fee, mocker) -> None: # Check the 2 filled orders equal the above amount assert pytest.approx(trade.orders[1].amount) == 30.150753768 assert pytest.approx(trade.orders[-1].amount) == 61.538461232 + + +def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog) -> None: + default_conf_usdt['position_adjustment_enable'] = True + + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker_usdt, + get_fee=fee, + amount_to_precision=lambda s, x, y: y, + price_to_precision=lambda s, x, y: y, + get_min_pair_stake_amount=MagicMock(return_value=10), + ) + + patch_get_signal(freqtrade) + freqtrade.enter_positions() + + assert len(Trade.get_trades().all()) == 1 + trade = Trade.get_trades().first() + assert len(trade.orders) == 1 + assert pytest.approx(trade.stake_amount) == 60 + assert pytest.approx(trade.amount) == 30.0 + assert trade.open_rate == 2.0 + + # Too small size + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-59) + freqtrade.process() + trade = Trade.get_trades().first() + assert len(trade.orders) == 1 + assert pytest.approx(trade.stake_amount) == 60 + assert pytest.approx(trade.amount) == 30.0 + assert log_has_re("Remaining amount of 1.6.* would be too small.", caplog) + + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-20) + + freqtrade.process() + trade = Trade.get_trades().first() + assert len(trade.orders) == 2 + assert trade.orders[-1].ft_order_side == 'sell' + assert pytest.approx(trade.stake_amount) == 40.198 + assert pytest.approx(trade.amount) == 20.099 + assert trade.open_rate == 2.0 + assert trade.is_open + caplog.clear() + + # Sell more than what we got (we got ~20 coins left) + # First adjusts the amount to 20 - then rejects. + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-50) + freqtrade.process() + assert log_has_re("Adjusting amount to trade.amount as it is higher.*", caplog) + assert log_has_re("Remaining amount of 0.0 would be too small.", caplog) + trade = Trade.get_trades().first() + assert len(trade.orders) == 2 + assert trade.orders[-1].ft_order_side == 'sell' + assert pytest.approx(trade.stake_amount) == 40.198 + assert trade.is_open diff --git a/tests/test_periodiccache.py b/tests/test_periodiccache.py index b2bd8ba2b..df05de4ef 100644 --- a/tests/test_periodiccache.py +++ b/tests/test_periodiccache.py @@ -1,6 +1,6 @@ import time_machine -from freqtrade.configuration import PeriodicCache +from freqtrade.util import PeriodicCache def test_ttl_cache(): diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 838c4c22a..2460fde68 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -99,7 +99,7 @@ def test_enter_exit_side(fee, is_short): @pytest.mark.usefixtures("init_persistence") -def test_set_stop_loss_isolated_liq(fee): +def test_set_stop_loss_liquidation(fee): trade = Trade( id=2, pair='ADA/USDT', @@ -115,73 +115,94 @@ def test_set_stop_loss_isolated_liq(fee): leverage=2.0, trading_mode=margin ) - trade.set_isolated_liq(0.09) + trade.set_liquidation_price(0.09) assert trade.liquidation_price == 0.09 assert trade.stop_loss is None assert trade.initial_stop_loss is None - trade._set_stop_loss(0.1, (1.0 / 9.0)) + trade.adjust_stop_loss(2.0, 0.2, True) assert trade.liquidation_price == 0.09 - assert trade.stop_loss == 0.1 - assert trade.initial_stop_loss == 0.1 + assert trade.stop_loss == 1.8 + assert trade.initial_stop_loss == 1.8 - trade.set_isolated_liq(0.08) + trade.set_liquidation_price(0.08) assert trade.liquidation_price == 0.08 - assert trade.stop_loss == 0.1 - assert trade.initial_stop_loss == 0.1 + assert trade.stop_loss == 1.8 + assert trade.initial_stop_loss == 1.8 - trade.set_isolated_liq(0.11) - trade._set_stop_loss(0.1, 0) + trade.set_liquidation_price(0.11) + trade.adjust_stop_loss(2.0, 0.2) assert trade.liquidation_price == 0.11 - assert trade.stop_loss == 0.11 - assert trade.initial_stop_loss == 0.1 + # Stoploss does not change from liquidation price + assert trade.stop_loss == 1.8 + assert trade.initial_stop_loss == 1.8 # lower stop doesn't move stoploss - trade._set_stop_loss(0.1, 0) + trade.adjust_stop_loss(1.8, 0.2) assert trade.liquidation_price == 0.11 - assert trade.stop_loss == 0.11 - assert trade.initial_stop_loss == 0.1 + assert trade.stop_loss == 1.8 + assert trade.initial_stop_loss == 1.8 + + # higher stop does move stoploss + trade.adjust_stop_loss(2.1, 0.1) + assert trade.liquidation_price == 0.11 + assert pytest.approx(trade.stop_loss) == 1.994999 + assert trade.initial_stop_loss == 1.8 + assert trade.stoploss_or_liquidation == trade.stop_loss trade.stop_loss = None trade.liquidation_price = None trade.initial_stop_loss = None + trade.initial_stop_loss_pct = None - trade._set_stop_loss(0.07, 0) + trade.adjust_stop_loss(2.0, 0.1, True) assert trade.liquidation_price is None - assert trade.stop_loss == 0.07 - assert trade.initial_stop_loss == 0.07 + assert trade.stop_loss == 1.9 + assert trade.initial_stop_loss == 1.9 + assert trade.stoploss_or_liquidation == 1.9 trade.is_short = True trade.recalc_open_trade_value() trade.stop_loss = None trade.initial_stop_loss = None + trade.initial_stop_loss_pct = None - trade.set_isolated_liq(0.09) - assert trade.liquidation_price == 0.09 + trade.set_liquidation_price(3.09) + assert trade.liquidation_price == 3.09 assert trade.stop_loss is None assert trade.initial_stop_loss is None - trade._set_stop_loss(0.08, (1.0 / 9.0)) - assert trade.liquidation_price == 0.09 - assert trade.stop_loss == 0.08 - assert trade.initial_stop_loss == 0.08 + trade.adjust_stop_loss(2.0, 0.2) + assert trade.liquidation_price == 3.09 + assert trade.stop_loss == 2.2 + assert trade.initial_stop_loss == 2.2 + assert trade.stoploss_or_liquidation == 2.2 - trade.set_isolated_liq(0.1) - assert trade.liquidation_price == 0.1 - assert trade.stop_loss == 0.08 - assert trade.initial_stop_loss == 0.08 + trade.set_liquidation_price(3.1) + assert trade.liquidation_price == 3.1 + assert trade.stop_loss == 2.2 + assert trade.initial_stop_loss == 2.2 + assert trade.stoploss_or_liquidation == 2.2 - trade.set_isolated_liq(0.07) - trade._set_stop_loss(0.1, (1.0 / 8.0)) - assert trade.liquidation_price == 0.07 - assert trade.stop_loss == 0.07 - assert trade.initial_stop_loss == 0.08 + trade.set_liquidation_price(3.8) + assert trade.liquidation_price == 3.8 + # Stoploss does not change from liquidation price + assert trade.stop_loss == 2.2 + assert trade.initial_stop_loss == 2.2 # Stop doesn't move stop higher - trade._set_stop_loss(0.1, (1.0 / 9.0)) - assert trade.liquidation_price == 0.07 - assert trade.stop_loss == 0.07 - assert trade.initial_stop_loss == 0.08 + trade.adjust_stop_loss(2.0, 0.3) + assert trade.liquidation_price == 3.8 + assert trade.stop_loss == 2.2 + assert trade.initial_stop_loss == 2.2 + + # Stoploss does move lower + trade.set_liquidation_price(1.5) + trade.adjust_stop_loss(1.8, 0.1) + assert trade.liquidation_price == 1.5 + assert pytest.approx(trade.stop_loss) == 1.89 + assert trade.initial_stop_loss == 2.2 + assert trade.stoploss_or_liquidation == 1.5 @pytest.mark.parametrize('exchange,is_short,lev,minutes,rate,interest,trading_mode', [ @@ -479,7 +500,7 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ assert trade.close_profit is None assert trade.close_date is None - trade.open_order_id = 'something' + trade.open_order_id = enter_order['id'] oobj = Order.parse_from_ccxt_object(enter_order, 'ADA/USDT', entry_side) trade.orders.append(oobj) trade.update_trade(oobj) @@ -494,7 +515,7 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ caplog) caplog.clear() - trade.open_order_id = 'something' + trade.open_order_id = enter_order['id'] time_machine.move_to("2022-03-31 21:45:05 +00:00") oobj = Order.parse_from_ccxt_object(exit_order, 'ADA/USDT', exit_side) trade.orders.append(oobj) @@ -529,7 +550,7 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, leverage=1.0, ) - trade.open_order_id = 'something' + trade.open_order_id = 'mocked_market_buy' oobj = Order.parse_from_ccxt_object(market_buy_order_usdt, 'ADA/USDT', 'buy') trade.orders.append(oobj) trade.update_trade(oobj) @@ -544,7 +565,7 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, caplog.clear() trade.is_open = True - trade.open_order_id = 'something' + trade.open_order_id = 'mocked_market_sell' oobj = Order.parse_from_ccxt_object(market_sell_order_usdt, 'ADA/USDT', 'sell') trade.orders.append(oobj) trade.update_trade(oobj) @@ -609,14 +630,14 @@ def test_calc_open_close_trade_price( trade.open_rate = 2.0 trade.close_rate = 2.2 trade.recalc_open_trade_value() - assert isclose(trade._calc_open_trade_value(), open_value) + assert isclose(trade._calc_open_trade_value(trade.amount, trade.open_rate), open_value) assert isclose(trade.calc_close_trade_value(trade.close_rate), close_value) assert isclose(trade.calc_profit(trade.close_rate), round(profit, 8)) assert pytest.approx(trade.calc_profit_ratio(trade.close_rate)) == profit_ratio @pytest.mark.usefixtures("init_persistence") -def test_trade_close(limit_buy_order_usdt, limit_sell_order_usdt, fee): +def test_trade_close(fee): trade = Trade( pair='ADA/USDT', stake_amount=60.0, @@ -794,7 +815,7 @@ def test_calc_open_trade_value( trade.update_trade(oobj) # Buy @ 2.0 # Get the open rate price with the standard fee rate - assert trade._calc_open_trade_value() == result + assert trade._calc_open_trade_value(trade.amount, trade.open_rate) == result @pytest.mark.parametrize( @@ -884,7 +905,7 @@ def test_calc_close_trade_price( ('binance', False, 1, 1.9, 0.003, -3.3209999, -0.055211970, spot, 0), ('binance', False, 1, 2.2, 0.003, 5.6520000, 0.093965087, spot, 0), - # # FUTURES, funding_fee=1 + # FUTURES, funding_fee=1 ('binance', False, 1, 2.1, 0.0025, 3.6925, 0.06138819, futures, 1), ('binance', False, 3, 2.1, 0.0025, 3.6925, 0.18416458, futures, 1), ('binance', True, 1, 2.1, 0.0025, -2.3074999, -0.03855472, futures, 1), @@ -1170,6 +1191,11 @@ def test_calc_profit( assert pytest.approx(trade.calc_profit(rate=close_rate)) == round(profit, 8) assert pytest.approx(trade.calc_profit_ratio(rate=close_rate)) == round(profit_ratio, 8) + assert pytest.approx(trade.calc_profit(close_rate, trade.amount, + trade.open_rate)) == round(profit, 8) + assert pytest.approx(trade.calc_profit_ratio(close_rate, trade.amount, + trade.open_rate)) == round(profit_ratio, 8) + def test_migrate_new(mocker, default_conf, fee, caplog): """ @@ -1361,7 +1387,9 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert log_has("trying trades_bak2", caplog) assert log_has("Running database migration for trades - backup: trades_bak2, orders_bak0", caplog) - assert trade.open_trade_value == trade._calc_open_trade_value() + assert log_has("Database migration finished.", caplog) + assert pytest.approx(trade.open_trade_value) == trade._calc_open_trade_value( + trade.amount, trade.open_rate) assert trade.close_profit_abs is None orders = trade.orders @@ -1537,26 +1565,26 @@ def test_adjust_stop_loss(fee): # Get percent of profit with a custom rate (Higher than open rate) trade.adjust_stop_loss(1.3, -0.1) - assert round(trade.stop_loss, 8) == 1.17 + assert pytest.approx(trade.stop_loss) == 1.17 assert trade.stop_loss_pct == -0.1 assert trade.initial_stop_loss == 0.95 assert trade.initial_stop_loss_pct == -0.05 # current rate lower again ... should not change trade.adjust_stop_loss(1.2, 0.1) - assert round(trade.stop_loss, 8) == 1.17 + assert pytest.approx(trade.stop_loss) == 1.17 assert trade.initial_stop_loss == 0.95 assert trade.initial_stop_loss_pct == -0.05 # current rate higher... should raise stoploss trade.adjust_stop_loss(1.4, 0.1) - assert round(trade.stop_loss, 8) == 1.26 + assert pytest.approx(trade.stop_loss) == 1.26 assert trade.initial_stop_loss == 0.95 assert trade.initial_stop_loss_pct == -0.05 # Initial is true but stop_loss set - so doesn't do anything trade.adjust_stop_loss(1.7, 0.1, True) - assert round(trade.stop_loss, 8) == 1.26 + assert pytest.approx(trade.stop_loss) == 1.26 assert trade.initial_stop_loss == 0.95 assert trade.initial_stop_loss_pct == -0.05 assert trade.stop_loss_pct == -0.1 @@ -1609,9 +1637,10 @@ def test_adjust_stop_loss_short(fee): assert trade.initial_stop_loss == 1.05 assert trade.initial_stop_loss_pct == -0.05 assert trade.stop_loss_pct == -0.1 - trade.set_isolated_liq(0.63) + # Liquidation price is lower than stoploss - so liquidation would trigger first. + trade.set_liquidation_price(0.63) trade.adjust_stop_loss(0.59, -0.1) - assert trade.stop_loss == 0.63 + assert trade.stop_loss == 0.649 assert trade.liquidation_price == 0.63 @@ -1660,6 +1689,7 @@ def test_get_open(fee, is_short, use_db): create_mock_trades(fee, is_short, use_db) assert len(Trade.get_open_trades()) == 4 + assert Trade.get_open_trade_count() == 4 Trade.use_db = True @@ -1672,6 +1702,7 @@ def test_get_open_lev(fee, use_db): create_mock_trades_with_leverage(fee, use_db) assert len(Trade.get_open_trades()) == 5 + assert Trade.get_open_trade_count() == 5 Trade.use_db = True @@ -1722,6 +1753,7 @@ def test_to_json(fee): 'stake_amount': 0.001, 'trade_duration': None, 'trade_duration_s': None, + 'realized_profit': 0.0, 'close_profit': None, 'close_profit_pct': None, 'close_profit_abs': None, @@ -1798,6 +1830,7 @@ def test_to_json(fee): 'initial_stop_loss_abs': None, 'initial_stop_loss_pct': None, 'initial_stop_loss_ratio': None, + 'realized_profit': 0.0, 'close_profit': None, 'close_profit_pct': None, 'close_profit_abs': None, @@ -1855,6 +1888,7 @@ def test_stoploss_reinitialization(default_conf, fee): assert trade.initial_stop_loss == 0.95 assert trade.initial_stop_loss_pct == -0.05 Trade.query.session.add(trade) + Trade.commit() # Lower stoploss Trade.stoploss_reinitialization(0.06) @@ -1916,6 +1950,7 @@ def test_stoploss_reinitialization_leverage(default_conf, fee): assert trade.initial_stop_loss == 0.98 assert trade.initial_stop_loss_pct == -0.1 Trade.query.session.add(trade) + Trade.commit() # Lower stoploss Trade.stoploss_reinitialization(0.15) @@ -1977,6 +2012,7 @@ def test_stoploss_reinitialization_short(default_conf, fee): assert trade.initial_stop_loss == 1.02 assert trade.initial_stop_loss_pct == -0.1 Trade.query.session.add(trade) + Trade.commit() # Lower stoploss Trade.stoploss_reinitialization(-0.15) trades = Trade.get_open_trades() @@ -2009,10 +2045,10 @@ def test_stoploss_reinitialization_short(default_conf, fee): assert trade_adj.initial_stop_loss == 1.01 assert trade_adj.initial_stop_loss_pct == -0.05 # Stoploss can't go above liquidation price - trade_adj.set_isolated_liq(0.985) + trade_adj.set_liquidation_price(0.985) trade.adjust_stop_loss(0.9799, -0.05) - assert trade_adj.stop_loss == 0.985 - assert trade_adj.stop_loss == 0.985 + assert trade_adj.stop_loss == 0.989699 + assert trade_adj.liquidation_price == 0.985 def test_update_fee(fee): @@ -2240,7 +2276,7 @@ def test_update_order_from_ccxt(caplog): 'symbol': 'ADA/USDT', 'type': 'limit', 'price': 1234.5, - 'amount': 20.0, + 'amount': 20.0, 'filled': 9, 'remaining': 11, 'status': 'open', @@ -2346,6 +2382,7 @@ def test_Trade_object_idem(): 'delete', 'session', 'commit', + 'rollback', 'query', 'open_date', 'get_best_pair', @@ -2399,7 +2436,7 @@ def test_recalc_trade_from_orders(fee): ) assert fee.return_value == 0.0025 - assert trade._calc_open_trade_value() == o1_trade_val + assert trade._calc_open_trade_value(trade.amount, trade.open_rate) == o1_trade_val assert trade.amount == o1_amount assert trade.stake_amount == o1_cost assert trade.open_rate == o1_rate @@ -2511,7 +2548,8 @@ def test_recalc_trade_from_orders(fee): assert pytest.approx(trade.fee_open_cost) == o1_fee_cost + o2_fee_cost + o3_fee_cost assert pytest.approx(trade.open_trade_value) == o1_trade_val + o2_trade_val + o3_trade_val - # Just to make sure sell orders are ignored, let's calculate one more time. + # Just to make sure full sell orders are ignored, let's calculate one more time. + sell1 = Order( ft_order_side='sell', ft_pair=trade.pair, @@ -2673,7 +2711,7 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): assert trade.open_trade_value == 2 * o1_trade_val assert trade.nr_of_successful_entries == 2 - # Just to make sure exit orders are ignored, let's calculate one more time. + # Reduce position - this will reduce amount again. sell1 = Order( ft_order_side=exit_side, ft_pair=trade.pair, @@ -2684,7 +2722,7 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): side=exit_side, price=4, average=3, - filled=2, + filled=o1_amount, remaining=1, cost=5, order_date=trade.open_date, @@ -2693,11 +2731,11 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): trade.orders.append(sell1) trade.recalc_trade_from_orders() - assert trade.amount == 2 * o1_amount - assert trade.stake_amount == 2 * o1_amount + assert trade.amount == o1_amount + assert trade.stake_amount == o1_amount assert trade.open_rate == o1_rate - assert trade.fee_open_cost == 2 * o1_fee_cost - assert trade.open_trade_value == 2 * o1_trade_val + assert trade.fee_open_cost == o1_fee_cost + assert trade.open_trade_value == o1_trade_val assert trade.nr_of_successful_entries == 2 # Check with 1 order @@ -2721,11 +2759,11 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): trade.recalc_trade_from_orders() # Calling recalc with single initial order should not change anything - assert trade.amount == 3 * o1_amount - assert trade.stake_amount == 3 * o1_amount + assert trade.amount == 2 * o1_amount + assert trade.stake_amount == 2 * o1_amount assert trade.open_rate == o1_rate - assert trade.fee_open_cost == 3 * o1_fee_cost - assert trade.open_trade_value == 3 * o1_trade_val + assert trade.fee_open_cost == 2 * o1_fee_cost + assert trade.open_trade_value == 2 * o1_trade_val assert trade.nr_of_successful_entries == 3 @@ -2793,3 +2831,143 @@ def test_order_to_ccxt(limit_buy_order_open): del raw_order['stopPrice'] del limit_buy_order_open['datetime'] assert raw_order == limit_buy_order_open + + +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize('data', [ + { + # tuple 1 - side, amount, price + # tuple 2 - amount, open_rate, stake_amount, cumulative_profit, realized_profit, rel_profit + 'orders': [ + (('buy', 100, 10), (100.0, 10.0, 1000.0, 0.0, None, None)), + (('buy', 100, 15), (200.0, 12.5, 2500.0, 0.0, None, None)), + (('sell', 50, 12), (150.0, 12.5, 1875.0, -25.0, -25.0, -0.04)), + (('sell', 100, 20), (50.0, 12.5, 625.0, 725.0, 750.0, 0.60)), + (('sell', 50, 5), (50.0, 12.5, 625.0, 350.0, -375.0, -0.60)), + ], + 'end_profit': 350.0, + 'end_profit_ratio': 0.14, + 'fee': 0.0, + }, + { + 'orders': [ + (('buy', 100, 10), (100.0, 10.0, 1000.0, 0.0, None, None)), + (('buy', 100, 15), (200.0, 12.5, 2500.0, 0.0, None, None)), + (('sell', 50, 12), (150.0, 12.5, 1875.0, -28.0625, -28.0625, -0.044788)), + (('sell', 100, 20), (50.0, 12.5, 625.0, 713.8125, 741.875, 0.59201995)), + (('sell', 50, 5), (50.0, 12.5, 625.0, 336.625, -377.1875, -0.60199501)), + ], + 'end_profit': 336.625, + 'end_profit_ratio': 0.1343142, + 'fee': 0.0025, + }, + { + 'orders': [ + (('buy', 100, 3), (100.0, 3.0, 300.0, 0.0, None, None)), + (('buy', 100, 7), (200.0, 5.0, 1000.0, 0.0, None, None)), + (('sell', 100, 11), (100.0, 5.0, 500.0, 596.0, 596.0, 1.189027)), + (('buy', 150, 15), (250.0, 11.0, 2750.0, 596.0, 596.0, 1.189027)), + (('sell', 100, 19), (150.0, 11.0, 1650.0, 1388.5, 792.5, 0.7186579)), + (('sell', 150, 23), (150.0, 11.0, 1650.0, 3175.75, 1787.25, 1.08048062)), + ], + 'end_profit': 3175.75, + 'end_profit_ratio': 0.9747170, + 'fee': 0.0025, + }, + { + # Test above without fees + 'orders': [ + (('buy', 100, 3), (100.0, 3.0, 300.0, 0.0, None, None)), + (('buy', 100, 7), (200.0, 5.0, 1000.0, 0.0, None, None)), + (('sell', 100, 11), (100.0, 5.0, 500.0, 600.0, 600.0, 1.2)), + (('buy', 150, 15), (250.0, 11.0, 2750.0, 600.0, 600.0, 1.2)), + (('sell', 100, 19), (150.0, 11.0, 1650.0, 1400.0, 800.0, 0.72727273)), + (('sell', 150, 23), (150.0, 11.0, 1650.0, 3200.0, 1800.0, 1.09090909)), + ], + 'end_profit': 3200.0, + 'end_profit_ratio': 0.98461538, + 'fee': 0.0, + }, + { + 'orders': [ + (('buy', 100, 8), (100.0, 8.0, 800.0, 0.0, None, None)), + (('buy', 100, 9), (200.0, 8.5, 1700.0, 0.0, None, None)), + (('sell', 100, 10), (100.0, 8.5, 850.0, 150.0, 150.0, 0.17647059)), + (('buy', 150, 11), (250.0, 10, 2500.0, 150.0, 150.0, 0.17647059)), + (('sell', 100, 12), (150.0, 10.0, 1500.0, 350.0, 200.0, 0.2)), + (('sell', 150, 14), (150.0, 10.0, 1500.0, 950.0, 600.0, 0.40)), + ], + 'end_profit': 950.0, + 'end_profit_ratio': 0.283582, + 'fee': 0.0, + }, +]) +def test_recalc_trade_from_orders_dca(data) -> None: + + pair = 'ETH/USDT' + trade = Trade( + id=2, + pair=pair, + stake_amount=1000, + open_rate=data['orders'][0][0][2], + amount=data['orders'][0][0][1], + is_open=True, + open_date=arrow.utcnow().datetime, + fee_open=data['fee'], + fee_close=data['fee'], + exchange='binance', + is_short=False, + leverage=1.0, + trading_mode=TradingMode.SPOT + ) + Trade.query.session.add(trade) + + for idx, (order, result) in enumerate(data['orders']): + amount = order[1] + price = order[2] + + order_obj = Order( + ft_order_side=order[0], + ft_pair=trade.pair, + order_id=f"order_{order[0]}_{idx}", + ft_is_open=False, + status="closed", + symbol=trade.pair, + order_type="market", + side=order[0], + price=price, + average=price, + filled=amount, + remaining=0, + cost=amount * price, + order_date=arrow.utcnow().shift(hours=-10 + idx).datetime, + order_filled_date=arrow.utcnow().shift(hours=-10 + idx).datetime, + ) + trade.orders.append(order_obj) + trade.recalc_trade_from_orders() + Trade.commit() + + orders1 = Order.query.all() + assert orders1 + assert len(orders1) == idx + 1 + + trade = Trade.query.first() + assert trade + assert len(trade.orders) == idx + 1 + if idx < len(data) - 1: + assert trade.is_open is True + assert trade.open_order_id is None + assert trade.amount == result[0] + assert trade.open_rate == result[1] + assert trade.stake_amount == result[2] + assert pytest.approx(trade.realized_profit) == result[3] + assert pytest.approx(trade.close_profit_abs) == result[4] + assert pytest.approx(trade.close_profit) == result[5] + + trade.close(price) + assert pytest.approx(trade.close_profit_abs) == data['end_profit'] + assert pytest.approx(trade.close_profit) == data['end_profit_ratio'] + assert not trade.is_open + trade = Trade.query.first() + assert trade + assert trade.open_order_id is None diff --git a/user_data/freqaimodels/.gitkeep b/user_data/freqaimodels/.gitkeep new file mode 100644 index 000000000..e69de29bb