diff --git a/.dockerignore b/.dockerignore index 889a4dfc7..abc5b82f0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ Dockerfile Dockerfile.armhf .dockerignore +docker/ .coveragerc .eggs .github diff --git a/.gitattributes b/.gitattributes index 00abd1d9d..81f6f422e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,3 @@ -*.py eol=lf -*.sh eol=lf -*.ps1 eol=crlf +*.py eol=lf +*.sh eol=lf +*.ps1 eol=crlf diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 7591c1fb0..5014de46a 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,5 +2,5 @@ blank_issues_enabled: false contact_links: - name: Discord Server - url: https://discord.gg/MA9v74M + url: https://discord.gg/p7nuUNVfP7 about: Ask a question or get community support from our Discord server diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a0837eb2..42959c3b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,7 @@ jobs: COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu run: | # Allow failure for coveralls - coveralls -v || true + coveralls || true - name: Backtesting run: | @@ -374,13 +374,6 @@ jobs: run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Build and test and push docker image - env: - IMAGE_NAME: freqtradeorg/freqtrade - BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} - run: | - build_helpers/publish_docker.sh - # We need docker experimental to pull the ARM image. - name: Switch docker to experimental run: | @@ -399,12 +392,12 @@ jobs: - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} - - name: Build Raspberry docker image + - name: Build and test and push docker images env: IMAGE_NAME: freqtradeorg/freqtrade - BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}_pi + BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} run: | - build_helpers/publish_docker_pi.sh + build_helpers/publish_docker_multi.sh - name: Slack Notification diff --git a/.travis.yml b/.travis.yml index 03a8df49b..4535c44cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,12 +46,6 @@ jobs: - script: mypy freqtrade scripts name: mypy - # - stage: docker - # if: branch in (master, develop, feat/improve_travis) AND (type in (push, cron)) - # script: - # - build_helpers/publish_docker.sh - # name: "Build and test and push docker image" - notifications: slack: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c29d6e632..040cf3e98 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ Few pointers for contributions: - New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR. - PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished). -If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. +If you are unsure, discuss the feature on our [discord server](https://discord.gg/p7nuUNVfP7), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. ## Getting started diff --git a/Dockerfile b/Dockerfile index 7d5afac9c..f2d7c8a40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,8 @@ ENV FT_APP_ENV="docker" # Prepare environment RUN mkdir /freqtrade \ - && apt update \ - && apt install -y sudo \ + && apt-get update \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ && apt-get clean \ && useradd -u 1000 -G sudo -U -m ftuser \ && chown ftuser:ftuser /freqtrade \ @@ -22,10 +22,10 @@ WORKDIR /freqtrade # Install dependencies FROM base as python-deps -RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev git \ - && apt-get clean \ - && pip install --upgrade pip +RUN apt-get update \ + && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \ + && apt-get clean \ + && pip install --upgrade pip # Install TA-lib COPY build_helpers/* /tmp/ @@ -49,7 +49,7 @@ USER ftuser # Install and execute COPY --chown=ftuser:ftuser . /freqtrade/ -RUN pip install -e . --user --no-cache-dir \ +RUN pip install -e . --user --no-cache-dir --no-build-isolation \ && mkdir /freqtrade/user_data/ \ && freqtrade install-ui diff --git a/README.md b/README.md index ab9597a77..c665508dd 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Telegram is not mandatory. However, this is a great way to control your bot. Mor - `/stop`: Stops the trader. - `/stopbuy`: Stop entering new trades. - `/status |[table]`: Lists all or specific open trades. -- `/profit`: Lists cumulative profit from all finished trades +- `/profit []`: Lists cumulative profit from all finished trades, over the last n days. - `/forcesell |all`: Instantly sells the given trade (Ignoring `minimum_roi`). - `/performance`: Show performance of each finished trade grouped by pair - `/balance`: Show account balance per currency. @@ -145,7 +145,7 @@ The project is currently setup in two main branches: For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel. -Please check out our [discord server](https://discord.gg/MA9v74M). +Please check out our [discord server](https://discord.gg/p7nuUNVfP7). You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). @@ -178,7 +178,7 @@ to understand the requirements before sending your pull-requests. Coding is not a necessity to contribute - maybe start with improving our documentation? Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. -**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/MA9v74M) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. +**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/p7nuUNVfP7) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. **Important:** Always create your PR against the `develop` branch, not `stable`. diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker_multi.sh similarity index 54% rename from build_helpers/publish_docker.sh rename to build_helpers/publish_docker_multi.sh index d987bcc69..a6b06ce7d 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker_multi.sh @@ -1,21 +1,48 @@ #!/bin/sh +# The below assumes a correctly setup docker buildx environment + # Replace / with _ to create a valid tag TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") TAG_PLOT=${TAG}_plot +TAG_PI="${TAG}_pi" + +PI_PLATFORM="linux/arm/v7" echo "Running for ${TAG}" +CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache # Add commit and commit_message to docker container echo "${GITHUB_SHA}" > freqtrade_commit if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" + # Build regular image docker build -t freqtrade:${TAG} . + # Build PI image + docker buildx build \ + --cache-to=type=registry,ref=${CACHE_TAG} \ + -f docker/Dockerfile.armhf \ + --platform ${PI_PLATFORM} \ + -t ${IMAGE_NAME}:${TAG_PI} --push . else echo "event ${GITHUB_EVENT_NAME}: building with cache" - # Pull last build to avoid rebuilding the whole image + # Build regular image docker pull ${IMAGE_NAME}:${TAG} docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . + + # Pull last build to avoid rebuilding the whole image + # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG} + docker buildx build \ + --cache-from=type=registry,ref=${CACHE_TAG} \ + --cache-to=type=registry,ref=${CACHE_TAG} \ + -f docker/Dockerfile.armhf \ + --platform ${PI_PLATFORM} \ + -t ${IMAGE_NAME}:${TAG_PI} --push . +fi + +if [ $? -ne 0 ]; then + echo "failed building multiarch images" + return 1 fi # Tag image for upload and next build step docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG @@ -24,11 +51,6 @@ docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t fre docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT -if [ $? -ne 0 ]; then - echo "failed building image" - return 1 -fi - # Run backtest docker run --rm -v $(pwd)/config_bittrex.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy @@ -37,23 +59,29 @@ if [ $? -ne 0 ]; then return 1 fi -if [ $? -ne 0 ]; then - echo "failed tagging image" - return 1 -fi - -# Tag as latest for develop builds -if [ "${TAG}" = "develop" ]; then - docker tag freqtrade:$TAG ${IMAGE_NAME}:latest -fi - -# Show all available images docker images docker push ${IMAGE_NAME} docker push ${IMAGE_NAME}:$TAG_PLOT docker push ${IMAGE_NAME}:$TAG + +# Create multiarch image +# Make sure that all images contained here are pushed to github first. +# Otherwise installation might fail. + +docker manifest create freqtradeorg/freqtrade:${TAG} ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI} +docker manifest push freqtradeorg/freqtrade:${TAG} + +# Tag as latest for develop builds +if [ "${TAG}" = "develop" ]; then + docker manifest create freqtradeorg/freqtrade:latest ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI} + docker manifest push freqtradeorg/freqtrade:latest +fi + + +docker images + if [ $? -ne 0 ]; then - echo "failed pushing repo" + echo "failed building image" return 1 fi diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh deleted file mode 100755 index 060b1deaf..000000000 --- a/build_helpers/publish_docker_pi.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh - -# The below assumes a correctly setup docker buildx environment - -# Replace / with _ to create a valid tag -TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") -PI_PLATFORM="linux/arm/v7" -echo "Running for ${TAG}" -CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache - -# Add commit and commit_message to docker container -echo "${GITHUB_SHA}" > freqtrade_commit - -if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then - echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" - docker buildx build \ - --cache-to=type=registry,ref=${CACHE_TAG} \ - -f Dockerfile.armhf \ - --platform ${PI_PLATFORM} \ - -t ${IMAGE_NAME}:${TAG} --push . -else - echo "event ${GITHUB_EVENT_NAME}: building with cache" - # Pull last build to avoid rebuilding the whole image - # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG} - docker buildx build \ - --cache-from=type=registry,ref=${CACHE_TAG} \ - --cache-to=type=registry,ref=${CACHE_TAG} \ - -f Dockerfile.armhf \ - --platform ${PI_PLATFORM} \ - -t ${IMAGE_NAME}:${TAG} --push . -fi - -if [ $? -ne 0 ]; then - echo "failed building image" - return 1 -fi diff --git a/config_full.json.example b/config_full.json.example index 24d364fdf..bc9f33f96 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -165,11 +165,22 @@ "startup": "on", "buy": "on", "buy_fill": "on", - "sell": "on", + "sell": { + "roi": "off", + "emergency_sell": "off", + "force_sell": "off", + "sell_signal": "off", + "trailing_stop_loss": "off", + "stop_loss": "off", + "stoploss_on_exchange": "off", + "custom_sell": "off" + }, "sell_fill": "on", "buy_cancel": "on", "sell_cancel": "on" - } + }, + "reload": true, + "balance_dust_level": 0.01 }, "api_server": { "enabled": false, diff --git a/docker/Dockerfile.aarch64 b/docker/Dockerfile.aarch64 deleted file mode 100644 index 71c10d949..000000000 --- a/docker/Dockerfile.aarch64 +++ /dev/null @@ -1,58 +0,0 @@ -FROM --platform=linux/arm64/v8 python:3.9.4-slim-buster as base - -# Setup env -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONFAULTHANDLER 1 -ENV PATH=/home/ftuser/.local/bin:$PATH -ENV FT_APP_ENV="docker" - -# Prepare environment -RUN mkdir /freqtrade \ - && apt-get update \ - && apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \ - && apt-get clean \ - && useradd -u 1000 -G sudo -U -m ftuser \ - && chown ftuser:ftuser /freqtrade \ - # Allow sudoers - && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers - -WORKDIR /freqtrade - -# Install dependencies -FROM base as python-deps -RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \ - && apt-get clean \ - && pip install --upgrade pip - -# Install TA-lib -COPY build_helpers/* /tmp/ -RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* -ENV LD_LIBRARY_PATH /usr/local/lib - -# Install dependencies -COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/ -USER ftuser -RUN pip install --user --no-cache-dir numpy \ - && pip install --user --no-cache-dir -r requirements-hyperopt.txt - -# Copy dependencies to runtime-image -FROM base as runtime-image -COPY --from=python-deps /usr/local/lib /usr/local/lib -ENV LD_LIBRARY_PATH /usr/local/lib - -COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local - -USER ftuser -# Install and execute -COPY --chown=ftuser:ftuser . /freqtrade/ - -RUN pip install -e . --user --no-cache-dir \ - && mkdir /freqtrade/user_data/ \ - && freqtrade install-ui - -ENTRYPOINT ["freqtrade"] -# Default to trade mode -CMD [ "trade" ] diff --git a/Dockerfile.armhf b/docker/Dockerfile.armhf similarity index 82% rename from Dockerfile.armhf rename to docker/Dockerfile.armhf index 2b3bca042..f9827774e 100644 --- a/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM --platform=linux/arm/v7 python:3.7.10-slim-buster as base +FROM python:3.7.10-slim-buster as base # Setup env ENV LANG C.UTF-8 @@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker" # Prepare environment RUN mkdir /freqtrade \ && apt-get update \ - && apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev \ && apt-get clean \ && useradd -u 1000 -G sudo -U -m ftuser \ && chown ftuser:ftuser /freqtrade \ @@ -22,7 +22,8 @@ WORKDIR /freqtrade # Install dependencies FROM base as python-deps -RUN apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 \ +RUN apt-get update \ + && apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 pkg-config cmake gcc \ && apt-get clean \ && pip install --upgrade pip \ && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf @@ -49,7 +50,7 @@ USER ftuser # Install and execute COPY --chown=ftuser:ftuser . /freqtrade/ -RUN pip install -e . --user --no-cache-dir \ +RUN pip install -e . --user --no-cache-dir --no-build-isolation\ && mkdir /freqtrade/user_data/ \ && freqtrade install-ui diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index c86978b80..35fd3de4a 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -289,7 +289,7 @@ Given the following result from hyperopt: ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 Buy hyperspace params: { 'adx-value': 44, diff --git a/docs/backtesting.md b/docs/backtesting.md index 2027c2079..4899b1dad 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -19,7 +19,7 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--enable-protections] [--dry-run-wallet DRY_RUN_WALLET] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] - [--export EXPORT] [--export-filename PATH] + [--export {none,trades}] [--export-filename PATH] optional arguments: -h, --help show this help message and exit @@ -63,8 +63,8 @@ optional arguments: name is injected into the filename (so `backtest- data.json` becomes `backtest-data- DefaultStrategy.json` - --export EXPORT Export backtest results, argument are: trades. - Example: `--export=trades` + --export {none,trades} + Export backtest results (default: trades). --export-filename PATH Save backtest results to the file with this filename. Requires `--export` to be set as well. Example: @@ -100,7 +100,7 @@ Strategy arguments: Now you have good Buy and Sell strategies and some historic data, you want to test it against real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). -Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHCLV) data from `user_data/data/` by default. +Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHLCV) data from `user_data/data/` by default. If no data is available for the exchange / pair / timeframe combination, backtesting will ask you to download them first using `freqtrade download-data`. For details on downloading, please refer to the [Data Downloading](data-download.md) section in the documentation. @@ -110,11 +110,16 @@ All profit calculations include fees, and freqtrade will use the exchange's defa !!! Warning "Using dynamic pairlists for backtesting" Using dynamic pairlists is possible, however it relies on the current market conditions - which will not reflect the historic status of the pairlist. - Also, when using pairlists other than StaticPairlist, reproducability of backtesting-results cannot be guaranteed. + Also, when using pairlists other than StaticPairlist, reproducibility of backtesting-results cannot be guaranteed. Please read the [pairlists documentation](plugins.md#pairlists) for more information. To achieve reproducible results, best generate a pairlist via the [`test-pairlist`](utils.md#test-pairlist) command and use that as static pairlist. +!!! Note + By default, Freqtrade will export backtesting results to `user_data/backtest_results`. + The exported trades can be used for [further analysis](#further-backtest-result-analysis) or can be used by the [plotting sub-command](plotting.md#plot-price-and-indicators) (`freqtrade plot-dataframe`) in the scripts directory. + + ### Starting balance Backtesting will require a starting balance, which can be provided as `--dry-run-wallet ` or `--starting-balance ` command line argument, or via `dry_run_wallet` configuration setting. @@ -174,13 +179,13 @@ Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies --- -Exporting trades to file +Prevent exporting trades to file ```bash -freqtrade backtesting --strategy backtesting --export trades --config config.json +freqtrade backtesting --strategy backtesting --export none --config config.json ``` -The exported trades can be used for [further analysis](#further-backtest-result-analysis), or can be used by the plotting script `plot_dataframe.py` in the scripts directory. +Only use this if you're sure you'll not want to plot or analyze your results further. --- @@ -279,7 +284,7 @@ A backtesting result will look like that: | Backtesting to | 2019-05-01 00:00:00 | | Max open trades | 3 | | | | -| Total trades | 429 | +| Total/Daily Avg Trades| 429 / 3.575 | | Starting balance | 0.01000000 BTC | | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | @@ -368,12 +373,11 @@ It contains some useful key metrics about performance of your strategy on backte | Backtesting to | 2019-05-01 00:00:00 | | Max open trades | 3 | | | | -| Total trades | 429 | +| Total/Daily Avg Trades| 429 / 3.575 | | Starting balance | 0.01000000 BTC | | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | -| Trades per day | 3.575 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | @@ -404,12 +408,11 @@ It contains some useful key metrics about performance of your strategy on backte - `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option). - `Max open trades`: Setting of `max_open_trades` (or `--max-open-trades`) - or number of pairs in the pairlist (whatever is lower). -- `Total trades`: Identical to the total trades of the backtest output table. +- `Total/Daily Avg Trades`: Identical to the total trades of the backtest output table / Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). - `Starting balance`: Start balance - as given by dry-run-wallet (config or command line). - `Final balance`: Final balance - starting balance + absolute profit. - `Absolute profit`: Profit made in stake currency. - `Total profit %`: Total profit. Aligned to the `TOTAL` row's `Tot Profit %` from the first table. Calculated as `(End capital − Starting capital) / Starting capital`. -- `Trades per day`: Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). - `Avg. stake amount`: Average stake amount, either `stake_amount` or the average when using dynamic stake amount. - `Total trade volume`: Volume generated on the exchange to reach the above profit. - `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`. @@ -441,6 +444,7 @@ Since backtesting lacks some detailed information about what happens within a ca - Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes - Low happens before high for stoploss, protecting capital first - Trailing stoploss + - Trailing Stoploss is only adjusted if it's below the candle's low (otherwise it would be triggered) - High happens first - adjusting stoploss - Low uses the adjusted stoploss (so sells with large high-low difference are backtested correctly) - ROI applies before trailing-stop, ensuring profits are "top-capped" at ROI if both ROI and trailing stop applies diff --git a/docs/configuration.md b/docs/configuration.md index eff8fe322..a2ed41b99 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -102,10 +102,11 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
**Datatype:** Positive Integer | `exchange.skip_pair_validation` | Skip pairlist validation on startup.
*Defaults to `false`
**Datatype:** Boolean | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.
*Defaults to `false`
**Datatype:** Boolean +| `exchange.log_responses` | Log relevant exchange responses. For debug mode only - use with care.
*Defaults to `false`
**Datatype:** Boolean | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean | `pairlists` | Define one or more pairlists to be used. [More information](plugins.md#pairlists-and-pairlist-handlers).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts -| `protections` | Define one or more protections to be used. [More information](plugins.md#protections). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** List of Dicts +| `protections` | Define one or more protections to be used. [More information](plugins.md#protections).
**Datatype:** List of Dicts | `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean | `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String @@ -140,7 +141,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi ### Parameters in the strategy -The following parameters can be set in either configuration file or strategy. +The following parameters can be set in configuration file or strategy. Values set in the configuration file always overwrite values set in the strategy. * `minimal_roi` @@ -156,7 +157,6 @@ Values set in the configuration file always overwrite values set in the strategy * `order_time_in_force` * `unfilledtimeout` * `disable_dataframe_checks` -* `protections` * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) * `sell_profit_offset` (ask_strategy) @@ -170,7 +170,7 @@ There are several methods to configure how much of the stake currency the bot wi #### Minimum trade stake The minimum stake amount will depend by exchange and pair, and is usually listed in the exchange support pages. -Assuming the minimum tradable amount for XRP/USD is 20 XRP (given by the exchange), and the price is 0.4$. +Assuming the minimum tradable amount for XRP/USD is 20 XRP (given by the exchange), and the price is 0.6$. The minimum stake amount to buy this pair is therefore `20 * 0.6 ~= 12`. This exchange has also a limit on USD - where all orders must be > 10$ - which however does not apply in this case. @@ -304,6 +304,9 @@ For example, if your strategy is using a 1h timeframe, and you only want to buy }, ``` +!!! Note + This setting resets with each new candle, so it will not prevent sticking-signals from executing on the 2nd or 3rd candle they're active. Best use a "trigger" selector for buy signals, which are only active for one candle. + ### Understand order_types The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`, `emergencysell`, `forcesell`, `forcebuy`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds. @@ -403,8 +406,8 @@ The possible values are: `gtc` (default), `fok` or `ioc`. ``` !!! Warning - This is an ongoing work. For now it is supported only for binance and only for buy orders. - Please don't change the default value unless you know what you are doing. + This is an ongoing work. For now it is supported only for binance. + Please don't change the default value unless you know what you are doing and have researched the impact of using different values. ### Exchange configuration @@ -500,7 +503,8 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode. * Wallets (`/balance`) are simulated based on `dry_run_wallet`. * Orders are simulated, and will not be posted to the exchange. -* Orders are assumed to fill immediately, and will never time out. +* Market orders fill based on orderbook volume the moment the order is placed. +* Limit orders fill once price reaches the defined level - or time out based on `unfilledtimeout` settings. * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled. * Open orders (not trades, which are stored in the database) are reset on bot restart. diff --git a/docs/developer.md b/docs/developer.md index 4b8c64530..d0731a233 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -2,7 +2,7 @@ This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running. -All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) where you can ask questions. +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/p7nuUNVfP7) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) where you can ask questions. ## Documentation diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 3a85aa885..cb66fc7e2 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -98,7 +98,7 @@ Create a new directory and place the [docker-compose file](https://raw.githubuse image: freqtradeorg/freqtrade:custom_arm64 build: context: . - dockerfile: "./docker/Dockerfile.aarch64" + dockerfile: "Dockerfile" ``` The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image. diff --git a/docs/exchanges.md b/docs/exchanges.md index 8797ade8c..e54f97714 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -14,11 +14,10 @@ Accounts having BNB accounts use this to pay for fees - if your first trade happ ### Binance sites -Binance has been split into 3, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. +Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. * [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`. * [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`. -* [binance.je](https://www.binance.je/) - Binance Jersey, trading fiat currencies. Use exchange id: `binanceje`. ## Kraken @@ -54,6 +53,9 @@ Due to the heavy rate-limiting applied by Kraken, the following configuration se Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faq.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy). +Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment. +Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected. + ### Restricted markets Bittrex split its exchange into US and International versions. diff --git a/docs/faq.md b/docs/faq.md index 7233a92fe..d015ae50e 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -136,6 +136,22 @@ On Windows, the `--logfile` option is also supported by Freqtrade and you can us > type \path\to\mylogfile.log | findstr "something" ``` +### Why does freqtrade not have GPU support? + +First of all, most indicator libraries don't have GPU support - as such, there would be little benefit for indicator calculations. +The GPU improvements would only apply to pandas-native calculations - or ones written by yourself. + +For hyperopt, freqtrade is using scikit-optimize, which is built on top of scikit-learn. +Their statement about GPU support is [pretty clear](https://scikit-learn.org/stable/faq.html#will-you-add-gpu-support). + +GPU's also are only good at crunching numbers (floating point operations). +For hyperopt, we need both number-crunching (find next parameters) and running python code (running backtesting). +As such, GPU's are not too well suited for most parts of hyperopt. + +The benefit of using GPU would therefore be pretty slim - and will not justify the complexity introduced by trying to add GPU support. + +There is however nothing preventing you from using GPU-enabled indicators within your strategy if you think you must have this - you will however probably be disappointed by the slim gain that will give you (compared to the complexity). + ## Hyperopt module ### How many epochs do I need to get a good Hyperopt result? @@ -156,7 +172,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD ### Why does it take a long time to run hyperopt? -* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/MA9v74M). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. +* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/p7nuUNVfP7). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers: diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 0d7b1cc32..a117ac1ce 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -237,9 +237,9 @@ class MyAwesomeStrategy(IStrategy): dataframe['macdhist'] = macd['macdhist'] bollinger = ta.BBANDS(dataframe, timeperiod=20, nbdevup=2.0, nbdevdn=2.0) - dataframe['bb_lowerband'] = boll['lowerband'] - dataframe['bb_middleband'] = boll['middleband'] - dataframe['bb_upperband'] = boll['upperband'] + dataframe['bb_lowerband'] = bollinger['lowerband'] + dataframe['bb_middleband'] = bollinger['middleband'] + dataframe['bb_upperband'] = bollinger['upperband'] return dataframe ``` @@ -491,7 +491,7 @@ Given the following result from hyperopt: ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 # Buy hyperspace params: buy_params = { @@ -532,7 +532,7 @@ If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'de ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 # ROI table: minimal_roi = { @@ -587,7 +587,7 @@ If you are optimizing stoploss values (i.e. if optimization search-space contain ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 # Buy hyperspace params: buy_params = { @@ -629,7 +629,7 @@ If you are optimizing trailing stop values (i.e. if optimization search-space co ``` Best result: - 45/100: 606 trades. Avg profit 1.04%. Total profit 0.31555614 BTC ( 630.48Σ%). Avg duration 150.3 mins. Objective: -1.10161 + 45/100: 606 trades. Avg profit 1.04%. Total profit 0.31555614 BTC ( 630.48%). Avg duration 150.3 mins. Objective: -1.10161 # Trailing stop: trailing_stop = True diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index ce0cc6e57..f19c5a181 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -122,8 +122,8 @@ The `max_price` setting removes pairs where the price is above the specified pri This option is disabled by default, and will only apply if set to > 0. The `max_value` setting removes pairs where the minimum value change is above a specified value. -This is useful when an exchange has unbalanced limits. For example, if step-size = 1 (so you can only buy 1, or 2, or 3, but not 1.1 Coins) - and the price is pretty high (like 20$) as the coin has risen sharply since the last limit adaption. -As a result of the above, you can only buy for 20$, or 40$ - but not for 25$. +This is useful when an exchange has unbalanced limits. For example, if step-size = 1 (so you can only buy 1, or 2, or 3, but not 1.1 Coins) - and the price is pretty high (like 20\$) as the coin has risen sharply since the last limit adaption. +As a result of the above, you can only buy for 20\$, or 40\$ - but not for 25\$. On exchanges that deduct fees from the receiving currency (e.g. FTX) - this can result in high value coins / amounts that are unsellable as the amount is slightly below the limit. The `low_price_ratio` setting removes pairs where a raise of 1 price unit (pip) is above the `low_price_ratio` ratio. diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 6bc57153e..3ea2dde61 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -8,7 +8,6 @@ All protection end times are rounded up to the next candle to avoid sudden, unex !!! Note Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy to improve performance. - To align your protection with your strategy, you can define protections in the strategy. !!! Tip Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term). @@ -47,16 +46,16 @@ This applies across all pairs, unless `only_per_pair` is set to true, which will The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. -```json -"protections": [ +``` python +protections = [ { "method": "StoplossGuard", "lookback_period_candles": 24, "trade_limit": 4, "stop_duration_candles": 4, - "only_per_pair": false + "only_per_pair": False } -], +] ``` !!! Note @@ -69,8 +68,8 @@ The below example stops trading for all pairs for 4 candles after the last trade The below sample stops trading for 12 candles if max-drawdown is > 20% considering all pairs - with a minimum of `trade_limit` trades - within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used. -```json -"protections": [ +``` python +protections = [ { "method": "MaxDrawdown", "lookback_period_candles": 48, @@ -78,7 +77,7 @@ The below sample stops trading for 12 candles if max-drawdown is > 20% consideri "stop_duration_candles": 12, "max_allowed_drawdown": 0.2 }, -], +] ``` #### Low Profit Pairs @@ -88,8 +87,8 @@ If that ratio is below `required_profit`, that pair will be locked for `stop_dur The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 candles. -```json -"protections": [ +``` python +protections = [ { "method": "LowProfitPairs", "lookback_period_candles": 6, @@ -97,7 +96,7 @@ The below example will stop trading a pair for 60 minutes if the pair does not h "stop_duration": 60, "required_profit": 0.02 } -], +] ``` #### Cooldown Period @@ -106,13 +105,13 @@ The below example will stop trading a pair for 60 minutes if the pair does not h The below example will stop trading a pair for 2 candles after closing a trade, allowing this pair to "cool down". -```json -"protections": [ +``` python +protections = [ { "method": "CooldownPeriod", "stop_duration_candles": 2 } -], +] ``` !!! Note @@ -132,46 +131,6 @@ The below example assumes a timeframe of 1 hour: * Locks all pairs that had 4 Trades within the last 6 hours (`6 * 1h candles`) with a combined profit ratio of below 0.02 (<2%) (`LowProfitPairs`). * Locks all pairs for 2 candles that had a profit of below 0.01 (<1%) within the last 24h (`24 * 1h candles`), a minimum of 4 trades. -```json -"timeframe": "1h", -"protections": [ - { - "method": "CooldownPeriod", - "stop_duration_candles": 5 - }, - { - "method": "MaxDrawdown", - "lookback_period_candles": 48, - "trade_limit": 20, - "stop_duration_candles": 4, - "max_allowed_drawdown": 0.2 - }, - { - "method": "StoplossGuard", - "lookback_period_candles": 24, - "trade_limit": 4, - "stop_duration_candles": 2, - "only_per_pair": false - }, - { - "method": "LowProfitPairs", - "lookback_period_candles": 6, - "trade_limit": 2, - "stop_duration_candles": 60, - "required_profit": 0.02 - }, - { - "method": "LowProfitPairs", - "lookback_period_candles": 24, - "trade_limit": 4, - "stop_duration_candles": 2, - "required_profit": 0.01 - } - ], -``` - -You can use the same in your strategy, the syntax is only slightly different: - ``` python from freqtrade.strategy import IStrategy diff --git a/docs/index.md b/docs/index.md index c2b6d5629..871172cfc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -76,7 +76,7 @@ Alternatively For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel. -Please check out our [discord server](https://discord.gg/MA9v74M). +Please check out our [discord server](https://discord.gg/p7nuUNVfP7). You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). diff --git a/docs/installation.md b/docs/installation.md index c19965a18..5c6ac001f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -60,7 +60,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces sudo apt-get update # install packages - sudo apt install -y python3-pip python3-venv python3-pandas git + sudo apt install -y python3-pip python3-venv python3-dev python3-pandas git ``` === "RaspberryPi/Raspbian" diff --git a/docs/overrides/main.html b/docs/overrides/main.html new file mode 100644 index 000000000..dfc5264be --- /dev/null +++ b/docs/overrides/main.html @@ -0,0 +1,68 @@ +{% extends "base.html" %} + + + +{% block site_nav %} + + + {% if nav %} + {% if page and page.meta and page.meta.hide %} + {% set hidden = "hidden" if "navigation" in page.meta.hide %} + {% endif %} + + {% endif %} + + + {% if page.toc and not "toc.integrate" in features %} + {% if page and page.meta and page.meta.hide %} + {% set hidden = "hidden" if "toc" in page.meta.hide %} + {% endif %} + + {% endif %} +{% endblock %} + +{% block footer %} + {{ super() }} + + + + + + + + + +{% endblock %} diff --git a/docs/partials/header.html b/docs/partials/header.html deleted file mode 100644 index 22132bc96..000000000 --- a/docs/partials/header.html +++ /dev/null @@ -1,72 +0,0 @@ -{#- -This file was automatically generated - do not edit --#} -{% set site_url = config.site_url | d(nav.homepage.url, true) | url %} -{% if not config.use_directory_urls and site_url[0] == site_url[-1] == "." %} -{% set site_url = site_url ~ "/index.html" %} -{% endif %} -
- - - - -
diff --git a/docs/plotting.md b/docs/plotting.md index 5d454c414..9fae38504 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -170,9 +170,15 @@ Additional features when using plot_config include: * Specify additional subplots * Specify indicator pairs to fill area in between -The sample plot configuration below specifies fixed colors for the indicators. Otherwise consecutive plots may produce different colorschemes each time, making comparisons difficult. +The sample plot configuration below specifies fixed colors for the indicators. Otherwise, consecutive plots may produce different color schemes each time, making comparisons difficult. It also allows multiple subplots to display both MACD and RSI at the same time. +Plot type can be configured using `type` key. Possible types are: +* `scatter` corresponding to `plotly.graph_objects.Scatter` class (default). +* `bar` corresponding to `plotly.graph_objects.Bar` class. + +Extra parameters to `plotly.graph_objects.*` constructor can be specified in `plotly` dict. + Sample configuration with inline comments explaining the process: ``` python @@ -198,7 +204,8 @@ Sample configuration with inline comments explaining the process: # Create subplot MACD "MACD": { 'macd': {'color': 'blue', 'fill_to': 'macdhist'}, - 'macdsignal': {'color': 'orange'} + 'macdsignal': {'color': 'orange'}, + 'macdhist': {'type': 'bar', 'plotly': {'opacity': 0.9}} }, # Additional subplot RSI "RSI": { @@ -213,6 +220,9 @@ Sample configuration with inline comments explaining the process: The above configuration assumes that `ema10`, `ema50`, `senkou_a`, `senkou_b`, `macd`, `macdsignal`, `macdhist` and `rsi` are columns in the DataFrame created by the strategy. +!!! Warning + `plotly` arguments are only supported with plotly library and will not work with freq-ui. + ## Plot profit ![plot-profit](assets/plot-profit.png) @@ -265,6 +275,7 @@ optional arguments: (backtest file)) Default: file -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`). + --auto-open Automatically open generated plot. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 89011272d..352103f8b 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,4 @@ -mkdocs-material==7.1.5 +mkdocs==1.2.1 +mkdocs-material==7.1.8 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 diff --git a/docs/stylesheets/ft.extra.css b/docs/stylesheets/ft.extra.css index 3369fa177..f9ad980c6 100644 --- a/docs/stylesheets/ft.extra.css +++ b/docs/stylesheets/ft.extra.css @@ -11,3 +11,18 @@ .rst-versions .rst-other-versions { color: white; } + + +#widget-wrapper { + height: calc(220px * 0.5625 + 18px); + width: 220px; + margin: 0 auto 16px auto; + border-style: solid; + border-color: var(--md-code-bg-color); + border-width: 1px; + border-radius: 5px; +} + +@media screen and (max-width: calc(76.25em - 1px)) { + #widget-wrapper { display: none; } +} diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 6174bf0fe..f5d9744b4 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -72,22 +72,32 @@ Example configuration showing the different settings: ``` json "telegram": { - "enabled": true, - "token": "your_telegram_token", - "chat_id": "your_telegram_chat_id", - "notification_settings": { - "status": "silent", - "warning": "on", - "startup": "off", - "buy": "silent", - "sell": "on", - "buy_cancel": "silent", - "sell_cancel": "on", - "buy_fill": "off", - "sell_fill": "off" - }, - "balance_dust_level": 0.01 - }, + "enabled": true, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id", + "notification_settings": { + "status": "silent", + "warning": "on", + "startup": "off", + "buy": "silent", + "sell": { + "roi": "silent", + "emergency_sell": "on", + "force_sell": "on", + "sell_signal": "silent", + "trailing_stop_loss": "on", + "stop_loss": "on", + "stoploss_on_exchange": "on", + "custom_sell": "silent" + }, + "buy_cancel": "silent", + "sell_cancel": "on", + "buy_fill": "off", + "sell_fill": "off" + }, + "reload": true, + "balance_dust_level": 0.01 +}, ``` `buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange. @@ -96,6 +106,7 @@ Example configuration showing the different settings: `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown. +`reload` allows you to disable reload-buttons on selected messages. ## Create a custom keyboard (command shortcut buttons) @@ -154,7 +165,7 @@ official commands. You can ask at any moment for help with `/help`. | `/count` | Displays number of trades used and available | `/locks` | Show currently locked pairs. | `/unlock ` | Remove the lock for this pair (or for this lock id). -| `/profit` | Display a summary of your profit/loss from close trades and some stats about your performance +| `/profit []` | Display a summary of your profit/loss from close trades and some stats about your performance, over the last n days (all trades by default) | `/forcesell ` | Instantly sells the given trade (Ignoring `minimum_roi`). | `/forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`). | `/forcebuy [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index ed0c70417..d76b655d7 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2021.5' +__version__ = '2021.6' if __version__ == 'develop': diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index ffd317799..7f4f7edd6 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -69,7 +69,7 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "timerange", "timeframe", "no_trades"] ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", - "trade_source", "timeframe"] + "trade_source", "timeframe", "plot_auto_open"] ARGS_INSTALL_UI = ["erase_ui_only"] diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 03d095e12..b3f912433 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -183,7 +183,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: """ Applies selections to the template and writes the result to config_path :param config_path: Path object for new config file. Should not exist yet - :param selecions: Dict containing selections taken by the user. + :param selections: Dict containing selections taken by the user. """ from jinja2.exceptions import TemplateNotFound try: @@ -213,7 +213,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: def start_new_config(args: Dict[str, Any]) -> None: """ Create a new strategy from a template - Asking the user questions to fill out the templateaccordingly. + Asking the user questions to fill out the template accordingly. """ config_path = Path(args['config'][0]) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index b583b47ba..b226415e7 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -167,8 +167,9 @@ AVAILABLE_CLI_OPTIONS = { ), "export": Arg( '--export', - help='Export backtest results, argument are: trades. ' - 'Example: `--export=trades`', + help='Export backtest results (default: trades).', + choices=constants.EXPORT_OPTIONS, + ), "exportfilename": Arg( '--export-filename', @@ -433,6 +434,11 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=750, ), + "plot_auto_open": Arg( + '--auto-open', + help='Automatically open generated plot.', + action='store_true', + ), "no_trades": Arg( '--no-trades', help='Skip using trades from backtesting file and DB.', diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 58191ddb4..3877e0801 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -8,11 +8,11 @@ from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 5ba3db9f9..cc0d653b9 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -8,9 +8,9 @@ import requests from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import render_template, render_template_with_fallback -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index e072e12cb..19337b407 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -6,9 +6,9 @@ from colorama import init as colorama_init from freqtrade.configuration import setup_utils_configuration from freqtrade.data.btanalysis import get_latest_hyperopt_file +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.optimize.optimize_reports import show_backtest_result -from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -67,7 +67,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: if epochs and not no_details: sorted_epochs = sorted(epochs, key=itemgetter('loss')) results = sorted_epochs[0] - HyperoptTools.print_epoch_details(results, total_epochs, print_json, no_header) + HyperoptTools.show_epoch_details(results, total_epochs, print_json, no_header) if epochs and export_csv: HyperoptTools.export_csv_file( @@ -132,8 +132,8 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: show_backtest_result(metrics['strategy_name'], metrics, metrics['stake_currency']) - HyperoptTools.print_epoch_details(val, total_epochs, print_json, no_header, - header_str="Epoch details") + HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header, + header_str="Epoch details") def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: @@ -197,8 +197,12 @@ def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List: return x['results_metrics']['duration'] else: # New mode - avg = x['results_metrics']['holding_avg'] - return avg.total_seconds() // 60 + if 'holding_avg_s' in x['results_metrics']: + avg = x['results_metrics']['holding_avg_s'] + return avg // 60 + raise OperationalException( + "Holding-average not available. Please omit the filter on average time, " + "or rerun hyperopt with this version") if filteroptions['filter_min_avg_time'] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index fa4bc1066..cd26aa60e 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -1,7 +1,6 @@ import csv import logging import sys -from collections import OrderedDict from pathlib import Path from typing import Any, Dict, List @@ -12,11 +11,11 @@ from tabulate import tabulate from freqtrade.configuration import setup_utils_configuration from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active, validate_exchanges from freqtrade.misc import plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -54,15 +53,21 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: reset = '' names = [s['name'] for s in objs] - objss_to_print = [{ + objs_to_print = [{ 'name': s['name'] if s['name'] else "--", 'location': s['location'].name, 'status': (red + "LOAD FAILED" + reset if s['class'] is None else "OK" if names.count(s['name']) == 1 else yellow + "DUPLICATE NAME" + reset) } for s in objs] - - print(tabulate(objss_to_print, headers='keys', tablefmt='psql', stralign='right')) + for idx, s in enumerate(objs): + if 'hyperoptable' in s: + objs_to_print[idx].update({ + 'hyperoptable': "Yes" if s['hyperoptable']['count'] > 0 else "No", + 'buy-Params': len(s['hyperoptable'].get('buy', [])), + 'sell-Params': len(s['hyperoptable'].get('sell', [])), + }) + print(tabulate(objs_to_print, headers='keys', tablefmt='psql', stralign='right')) def start_list_strategies(args: Dict[str, Any]) -> None: @@ -75,6 +80,11 @@ def start_list_strategies(args: Dict[str, Any]) -> None: strategy_objs = StrategyResolver.search_all_objects(directory, not args['print_one_column']) # Sort alphabetically strategy_objs = sorted(strategy_objs, key=lambda x: x['name']) + for obj in strategy_objs: + if obj['class']: + obj['hyperoptable'] = obj['class'].detect_all_parameters() + else: + obj['hyperoptable'] = {'count': 0} if args['print_one_column']: print('\n'.join([s['name'] for s in strategy_objs])) @@ -143,7 +153,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: pairs_only=pairs_only, active_only=active_only) # Sort the pairs/markets by symbol - pairs = OrderedDict(sorted(pairs.items())) + pairs = dict(sorted(pairs.items())) except Exception as e: raise OperationalException(f"Cannot get markets. Reason: {e}") from e diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 6323bc2b1..08174bde6 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -3,9 +3,9 @@ from typing import Any, Dict from freqtrade import constants from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import round_coin_value -from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -15,6 +15,7 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[ """ Prepare the configuration for the Hyperopt module :param args: Cli args from Arguments() + :param method: Bot running mode :return: Configuration """ config = setup_utils_configuration(args, method) diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index 0661cd03c..9f7a5958e 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -4,8 +4,8 @@ from typing import Any, Dict import rapidjson from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.resolvers import ExchangeResolver -from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -31,7 +31,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: results[curr] = pairlists.whitelist for curr, pairlist in results.items(): - if not args.get('print_one_column', False): + if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False): print(f"Pairs for {curr}: ") if args.get('print_one_column', False): diff --git a/freqtrade/commands/plot_commands.py b/freqtrade/commands/plot_commands.py index 5e547acb0..73a429a28 100644 --- a/freqtrade/commands/plot_commands.py +++ b/freqtrade/commands/plot_commands.py @@ -1,8 +1,8 @@ from typing import Any, Dict from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode def validate_plot_args(args: Dict[str, Any]) -> None: diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index 832caf153..f282447d4 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -1,10 +1,10 @@ import logging from typing import Any, Dict +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt, is_exchange_officially_supported, validate_exchange) -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index 3b0f778f4..22836ab19 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -1,7 +1,7 @@ import logging from typing import Any, Dict -from freqtrade.state import RunMode +from freqtrade.enums import RunMode from .check_exchange import remove_credentials from .config_validation import validate_config_consistency @@ -15,6 +15,7 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str """ Prepare the configuration for utils subcommands :param args: Cli args from Arguments() + :param method: Bot running mode :return: Configuration """ configuration = Configuration(args, method) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 31e38d572..3004d6bf7 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -6,8 +6,8 @@ from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match from freqtrade import constants +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index f6d0520c5..d2cc68c44 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -12,10 +12,10 @@ from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir from freqtrade.configuration.load_config import load_config_file, load_file +from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts -from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode logger = logging.getLogger(__name__) @@ -375,6 +375,9 @@ class Configuration: self._args_to_config(config, argname='plot_limit', logstring='Limiting plot to: {}') + self._args_to_config(config, argname='plot_auto_open', + logstring='Parameter --auto-open detected.') + self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') @@ -457,7 +460,7 @@ class Configuration: pairs_file = Path(self.args["pairs_file"]) logger.info(f'Reading pairs file "{pairs_file}".') # Download pairs from the pairs file if no config is specified - # or if pairs file is specified explicitely + # or if pairs file is specified explicitly if not pairs_file.exists(): raise OperationalException(f'No pairs file found with path "{pairs_file}".') config['pairs'] = load_file(pairs_file) diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 1320a375f..27190d259 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -43,7 +43,7 @@ def load_file(path: Path) -> Dict[str, Any]: with path.open('r') as file: config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE) except FileNotFoundError: - raise OperationalException(f'File file "{path}" not found!') + raise OperationalException(f'File "{path}" not found!') return config diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 5ec60eb59..6ee4047d2 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -12,6 +12,7 @@ PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec TIMEOUT_UNITS = ['minutes', 'seconds'] +EXPORT_OPTIONS = ['none', 'trades'] DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' @@ -61,7 +62,7 @@ DUST_PER_COIN = { } -# Soure files with destination directories within user-directory +# Source files with destination directories within user-directory USER_DATA_FILES = { 'sample_strategy.py': USERPATH_STRATEGIES, 'sample_hyperopt_advanced.py': USERPATH_HYPEROPTS, @@ -260,7 +261,13 @@ CONF_SCHEMA = { 'enum': TELEGRAM_SETTING_OPTIONS, 'default': 'off' }, - 'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'sell': { + 'type': ['string', 'object'], + 'additionalProperties': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS + } + }, 'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, 'sell_fill': { 'type': 'string', @@ -268,7 +275,8 @@ CONF_SCHEMA = { 'default': 'off' }, } - } + }, + 'reload': {'type': 'boolean'}, }, 'required': ['enabled', 'token', 'chat_id'], }, @@ -302,6 +310,7 @@ CONF_SCHEMA = { 'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password'] }, 'db_url': {'type': 'string'}, + 'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'forcebuy_enable': {'type': 'boolean'}, 'disable_dataframe_checks': {'type': 'boolean'}, diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index ffee0c52c..040f58d62 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -49,7 +49,7 @@ def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *, fill_missing: bool = True, drop_incomplete: bool = True) -> DataFrame: """ - Clense a OHLCV dataframe by + Cleanse a OHLCV dataframe by * Grouping it by date (removes duplicate tics) * dropping last candles if requested * Filling up missing data (if requested) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 1a86eece5..391ed4587 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -12,9 +12,9 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe from freqtrade.data.history import load_pair_history +from freqtrade.enums import RunMode from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.exchange import Exchange -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index e80cfeba2..dd60530aa 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -52,8 +52,8 @@ class HDF5DataHandler(IDataHandler): """ Store data in hdf5 file. :param pair: Pair - used to generate filename - :timeframe: Timeframe - used to generate filename - :data: Dataframe containing OHLCV data + :param timeframe: Timeframe - used to generate filename + :param data: Dataframe containing OHLCV data :return: None """ key = self._pair_ohlcv_key(pair, timeframe) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 86e9f75e6..eecb63d07 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -113,6 +113,7 @@ def refresh_data(datadir: Path, :param timeframe: Timeframe (e.g. "5m") :param pairs: List of pairs to load :param exchange: Exchange object + :param data_format: dataformat to use :param timerange: Limit data to be loaded to this timerange """ data_handler = get_datahandler(datadir, data_format) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 070d9039d..05052b2d7 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -49,8 +49,8 @@ class IDataHandler(ABC): """ Store ohlcv data. :param pair: Pair - used to generate filename - :timeframe: Timeframe - used to generate filename - :data: Dataframe containing OHLCV data + :param timeframe: Timeframe - used to generate filename + :param data: Dataframe containing OHLCV data :return: None """ @@ -245,8 +245,8 @@ def get_datahandler(datadir: Path, data_format: str = None, data_handler: IDataHandler = None) -> IDataHandler: """ :param datadir: Folder to save data - :data_format: dataformat to use - :data_handler: returns this datahandler if it exists or initializes a new one + :param data_format: dataformat to use + :param data_handler: returns this datahandler if it exists or initializes a new one """ if not data_handler: diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 301d228a8..990e75bd9 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -55,8 +55,8 @@ class JsonDataHandler(IDataHandler): format looks as follows: [[,,,,]] :param pair: Pair - used to generate filename - :timeframe: Timeframe - used to generate filename - :data: Dataframe containing OHLCV data + :param timeframe: Timeframe - used to generate filename + :param data: Dataframe containing OHLCV data :return: None """ filename = self._pair_data_filename(self._datadir, pair, timeframe) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 4bc0d660b..977b7e4ec 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -13,11 +13,11 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT from freqtrade.data.history import get_timerange, load_data, refresh_data +from freqtrade.enums import RunMode, SellType from freqtrade.exceptions import OperationalException from freqtrade.exchange.exchange import timeframe_to_seconds from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist -from freqtrade.state import RunMode -from freqtrade.strategy.interface import IStrategy, SellType +from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) @@ -301,7 +301,7 @@ class Edge: def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]: """ This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs - The calulation will be done per pair and per strategy. + The calculation will be done per pair and per strategy. """ # Removing pairs having less than min_trades_number min_trades_number = self.edge_config.get('min_trade_number', 10) diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py new file mode 100644 index 000000000..78163d86f --- /dev/null +++ b/freqtrade/enums/__init__.py @@ -0,0 +1,6 @@ +# flake8: noqa: F401 +from freqtrade.enums.rpcmessagetype import RPCMessageType +from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode +from freqtrade.enums.selltype import SellType +from freqtrade.enums.signaltype import SignalType +from freqtrade.enums.state import State diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py new file mode 100644 index 000000000..9c59f6108 --- /dev/null +++ b/freqtrade/enums/rpcmessagetype.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class RPCMessageType(Enum): + STATUS = 'status' + WARNING = 'warning' + STARTUP = 'startup' + BUY = 'buy' + BUY_FILL = 'buy_fill' + BUY_CANCEL = 'buy_cancel' + SELL = 'sell' + SELL_FILL = 'sell_fill' + SELL_CANCEL = 'sell_cancel' + + def __repr__(self): + return self.value + + def __str__(self): + return self.value diff --git a/freqtrade/state.py b/freqtrade/enums/runmode.py similarity index 68% rename from freqtrade/state.py rename to freqtrade/enums/runmode.py index 8ddff71d9..7826d1d0c 100644 --- a/freqtrade/state.py +++ b/freqtrade/enums/runmode.py @@ -1,23 +1,6 @@ -# pragma pylint: disable=too-few-public-methods - -""" -Bot state constant -""" from enum import Enum -class State(Enum): - """ - Bot application states - """ - RUNNING = 1 - STOPPED = 2 - RELOAD_CONFIG = 3 - - def __str__(self): - return f"{self.name.lower()}" - - class RunMode(Enum): """ Bot running mode (backtest, hyperopt, ...) diff --git a/freqtrade/enums/selltype.py b/freqtrade/enums/selltype.py new file mode 100644 index 000000000..015c30186 --- /dev/null +++ b/freqtrade/enums/selltype.py @@ -0,0 +1,20 @@ +from enum import Enum + + +class SellType(Enum): + """ + Enum to distinguish between sell reasons + """ + ROI = "roi" + STOP_LOSS = "stop_loss" + STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange" + TRAILING_STOP_LOSS = "trailing_stop_loss" + SELL_SIGNAL = "sell_signal" + FORCE_SELL = "force_sell" + EMERGENCY_SELL = "emergency_sell" + CUSTOM_SELL = "custom_sell" + NONE = "" + + def __str__(self): + # explicitly convert to String to help with exporting data. + return self.value diff --git a/freqtrade/enums/signaltype.py b/freqtrade/enums/signaltype.py new file mode 100644 index 000000000..d636f378a --- /dev/null +++ b/freqtrade/enums/signaltype.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class SignalType(Enum): + """ + Enum to distinguish between buy and sell signals + """ + BUY = "buy" + SELL = "sell" diff --git a/freqtrade/enums/state.py b/freqtrade/enums/state.py new file mode 100644 index 000000000..572e2299f --- /dev/null +++ b/freqtrade/enums/state.py @@ -0,0 +1,13 @@ +from enum import Enum + + +class State(Enum): + """ + Bot application states + """ + RUNNING = 1 + STOPPED = 2 + RELOAD_CONFIG = 3 + + def __str__(self): + return f"{self.name.lower()}" diff --git a/freqtrade/exceptions.py b/freqtrade/exceptions.py index caf970606..056be8720 100644 --- a/freqtrade/exceptions.py +++ b/freqtrade/exceptions.py @@ -47,7 +47,7 @@ class InvalidOrderException(ExchangeError): class RetryableOrderError(InvalidOrderException): """ This is returned when the order is not found. - This Error will be repeated with increasing backof (in line with DDosError). + This Error will be repeated with increasing backoff (in line with DDosError). """ @@ -75,6 +75,6 @@ class DDosProtection(TemporaryError): class StrategyError(FreqtradeException): """ - Errors with custom user-code deteced. + Errors with custom user-code detected. Usually caused by errors in the strategy. """ diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 0bcfa5e17..0c470cb24 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -68,6 +68,7 @@ class Binance(Exchange): amount=amount, price=rate, params=params) logger.info('stoploss limit order added for %s. ' 'stop price: %s. limit: %s', pair, stop_price, rate) + self._log_exchange_response('create_stoploss_order', order) return order except ccxt.InsufficientFunds as e: raise InsufficientFundsError( diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 93d8f7584..99430ea25 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -22,8 +22,8 @@ from pandas import DataFrame from freqtrade.constants import DEFAULT_AMOUNT_RESERVE_PERCENT, ListPairsWithTimeframes from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, - InvalidOrderException, OperationalException, RetryableOrderError, - TemporaryError) + InvalidOrderException, OperationalException, PricingError, + RetryableOrderError, TemporaryError) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, retrier_async) @@ -88,6 +88,11 @@ class Exchange: # Cache for 10 minutes ... self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=1, ttl=60 * 10) + # Cache values for 1800 to avoid frequent polling of the exchange for prices + # Caching only applies to RPC methods, so prices for open trades are still + # refreshed once every iteration. + self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) + self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) # Holds candles self._klines: Dict[Tuple[str, str], DataFrame] = {} @@ -99,6 +104,7 @@ class Exchange: logger.info('Instance is running with dry_run enabled') logger.info(f"Using CCXT {ccxt.__version__}") exchange_config = config['exchange'] + self.log_responses = exchange_config.get('log_responses', False) # Deep merge ft_has with default ft_has options self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default)) @@ -221,10 +227,15 @@ class Exchange: """exchange ccxt precisionMode""" return self._api.precisionMode + def _log_exchange_response(self, endpoint, response) -> None: + """ Log exchange responses """ + if self.log_responses: + logger.info(f"API {endpoint}: {response}") + def ohlcv_candle_limit(self, timeframe: str) -> int: """ Exchange ohlcv candle limit - Uses ohlcv_candle_limit_per_timeframe if the exchange has different limts + Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit :param timeframe: Timeframe to check :return: Candle limit as integer @@ -464,11 +475,11 @@ class Exchange: return endpoint in self._api.has and self._api.has[endpoint] def amount_to_precision(self, pair: str, amount: float) -> float: - ''' + """ Returns the amount to buy or sell to a precision the Exchange accepts Re-implementation of ccxt internal methods - ensuring we can test the result is correct based on our definitions. - ''' + """ if self.markets[pair]['precision']['amount']: amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE, precision=self.markets[pair]['precision']['amount'], @@ -478,14 +489,14 @@ class Exchange: return amount def price_to_precision(self, pair: str, price: float) -> float: - ''' + """ Returns the price rounded up to the precision the Exchange accepts. Partial Re-implementation of ccxt internal method decimal_to_precision(), which does not support rounding up TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and align with amount_to_precision(). Rounds up - ''' + """ if self.markets[pair]['precision']['price']: # price = float(decimal_to_precision(price, rounding_mode=ROUND, # precision=self.markets[pair]['precision']['price'], @@ -550,11 +561,13 @@ class Exchange: # See also #2575 at github. return max(min_stake_amounts) * amount_reserve_percent + # Dry-run methods + def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict[str, Any]: order_id = f'dry_run_{side}_{datetime.now().timestamp()}' _amount = self.amount_to_precision(pair, amount) - dry_order = { + dry_order: Dict[str, Any] = { 'id': order_id, 'symbol': pair, 'price': rate, @@ -570,26 +583,110 @@ class Exchange: 'fee': None, 'info': {} } - self._store_dry_order(dry_order, pair) + if dry_order["type"] in ["stop_loss_limit", "stop-loss-limit"]: + dry_order["info"] = {"stopPrice": dry_order["price"]} + + if dry_order["type"] == "market": + # Update market order pricing + average = self.get_dry_market_fill_price(pair, side, amount, rate) + dry_order.update({ + 'average': average, + 'cost': dry_order['amount'] * average, + }) + dry_order = self.add_dry_order_fee(pair, dry_order) + + dry_order = self.check_dry_limit_order_filled(dry_order) + + self._dry_run_open_orders[dry_order["id"]] = dry_order # Copy order and close it - so the returned order is open unless it's a market order return dry_order - def _store_dry_order(self, dry_order: Dict, pair: str) -> None: - closed_order = dry_order.copy() - if closed_order['type'] in ["market", "limit"]: - closed_order.update({ - 'status': 'closed', - 'filled': closed_order['amount'], - 'remaining': 0, - 'fee': { - 'currency': self.get_pair_quote_currency(pair), - 'cost': dry_order['cost'] * self.get_fee(pair), - 'rate': self.get_fee(pair) - } - }) - if closed_order["type"] in ["stop_loss_limit", "stop-loss-limit"]: - closed_order["info"].update({"stopPrice": closed_order["price"]}) - self._dry_run_open_orders[closed_order["id"]] = closed_order + def add_dry_order_fee(self, pair: str, dry_order: Dict[str, Any]) -> Dict[str, Any]: + dry_order.update({ + 'fee': { + 'currency': self.get_pair_quote_currency(pair), + 'cost': dry_order['cost'] * self.get_fee(pair), + 'rate': self.get_fee(pair) + } + }) + return dry_order + + def get_dry_market_fill_price(self, pair: str, side: str, amount: float, rate: float) -> float: + """ + Get the market order fill price based on orderbook interpolation + """ + if self.exchange_has('fetchL2OrderBook'): + ob = self.fetch_l2_order_book(pair, 20) + ob_type = 'asks' if side == 'buy' else 'bids' + + remaining_amount = amount + filled_amount = 0 + for book_entry in ob[ob_type]: + book_entry_price = book_entry[0] + book_entry_coin_volume = book_entry[1] + if remaining_amount > 0: + if remaining_amount < book_entry_coin_volume: + filled_amount += remaining_amount * book_entry_price + else: + filled_amount += book_entry_coin_volume * book_entry_price + remaining_amount -= book_entry_coin_volume + else: + break + else: + # If remaining_amount wasn't consumed completely (break was not called) + filled_amount += remaining_amount * book_entry_price + forecast_avg_filled_price = filled_amount / amount + return self.price_to_precision(pair, forecast_avg_filled_price) + + return rate + + def _is_dry_limit_order_filled(self, pair: str, side: str, limit: float) -> bool: + if not self.exchange_has('fetchL2OrderBook'): + return True + ob = self.fetch_l2_order_book(pair, 1) + if side == 'buy': + price = ob['asks'][0][0] + logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}") + if limit >= price: + return True + else: + price = ob['bids'][0][0] + logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}") + if limit <= price: + return True + return False + + def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]: + """ + Check dry-run limit order fill and update fee (if it filled). + """ + if order['status'] != "closed" and order['type'] in ["limit"]: + pair = order['symbol'] + if self._is_dry_limit_order_filled(pair, order['side'], order['price']): + order.update({ + 'status': 'closed', + 'filled': order['amount'], + 'remaining': 0, + }) + self.add_dry_order_fee(pair, order) + + return order + + def fetch_dry_run_order(self, order_id) -> Dict[str, Any]: + """ + Return dry-run order + Only call if running in dry-run mode. + """ + try: + order = self._dry_run_open_orders[order_id] + order = self.check_dry_limit_order_filled(order) + return order + except KeyError as e: + # Gracefully handle errors with dry-run orders. + raise InvalidOrderException( + f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + + # Order handling def create_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict: @@ -600,8 +697,10 @@ class Exchange: or self._api.options.get("createMarketBuyOrderRequiresPrice", False)) rate_for_order = self.price_to_precision(pair, rate) if needs_price else None - return self._api.create_order(pair, ordertype, side, - amount, rate_for_order, params) + order = self._api.create_order(pair, ordertype, side, + amount, rate_for_order, params) + self._log_exchange_response('create_order', order) + return order except ccxt.InsufficientFunds as e: raise InsufficientFundsError( @@ -667,6 +766,134 @@ class Exchange: raise OperationalException(f"stoploss is not implemented for {self.name}.") + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) + def fetch_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + return self.fetch_dry_run_order(order_id) + try: + order = self._api.fetch_order(order_id, pair) + self._log_exchange_response('fetch_order', order) + return order + except ccxt.OrderNotFound as e: + raise RetryableOrderError( + f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + # Assign method to fetch_stoploss_order to allow easy overriding in other classes + fetch_stoploss_order = fetch_order + + def fetch_order_or_stoploss_order(self, order_id: str, pair: str, + stoploss_order: bool = False) -> Dict: + """ + Simple wrapper calling either fetch_order or fetch_stoploss_order depending on + the stoploss_order parameter + :param order_id: OrderId to fetch order + :param pair: Pair corresponding to order_id + :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. + """ + if stoploss_order: + return self.fetch_stoploss_order(order_id, pair) + return self.fetch_order(order_id, pair) + + def check_order_canceled_empty(self, order: Dict) -> bool: + """ + Verify if an order has been cancelled without being partially filled + :param order: Order dict as returned from fetch_order() + :return: True if order has been cancelled without being filled, False otherwise. + """ + return (order.get('status') in ('closed', 'canceled', 'cancelled') + and order.get('filled') == 0.0) + + @retrier + def cancel_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + try: + order = self.fetch_dry_run_order(order_id) + + order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) + return order + except InvalidOrderException: + return {} + + try: + order = self._api.cancel_order(order_id, pair) + self._log_exchange_response('cancel_order', order) + return order + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not cancel order. Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + # Assign method to cancel_stoploss_order to allow easy overriding in other classes + cancel_stoploss_order = cancel_order + + def is_cancel_order_result_suitable(self, corder) -> bool: + if not isinstance(corder, dict): + return False + + required = ('fee', 'status', 'amount') + return all(k in corder for k in required) + + def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: + """ + Cancel order returning a result. + Creates a fake result if cancel order returns a non-usable result + and fetch_order does not work (certain exchanges don't return cancelled orders) + :param order_id: Orderid to cancel + :param pair: Pair corresponding to order_id + :param amount: Amount to use for fake response + :return: Result from either cancel_order if usable, or fetch_order + """ + try: + corder = self.cancel_order(order_id, pair) + if self.is_cancel_order_result_suitable(corder): + return corder + except InvalidOrderException: + logger.warning(f"Could not cancel order {order_id} for {pair}.") + try: + order = self.fetch_order(order_id, pair) + except InvalidOrderException: + logger.warning(f"Could not fetch cancelled order {order_id}.") + order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} + + return order + + def cancel_stoploss_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: + """ + Cancel stoploss order returning a result. + Creates a fake result if cancel order returns a non-usable result + and fetch_order does not work (certain exchanges don't return cancelled orders) + :param order_id: stoploss-order-id to cancel + :param pair: Pair corresponding to order_id + :param amount: Amount to use for fake response + :return: Result from either cancel_order if usable, or fetch_order + """ + corder = self.cancel_stoploss_order(order_id, pair) + if self.is_cancel_order_result_suitable(corder): + return corder + try: + order = self.fetch_stoploss_order(order_id, pair) + except InvalidOrderException: + logger.warning(f"Could not fetch cancelled stoploss order {order_id}.") + order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} + + return order + @retrier def get_balances(self) -> dict: @@ -713,6 +940,8 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + # Pricing info + @retrier def fetch_ticker(self, pair: str) -> dict: try: @@ -729,6 +958,265 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + @staticmethod + def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]], + range_required: bool = True): + """ + Get next greater value in the list. + Used by fetch_l2_order_book if the api only supports a limited range + """ + if not limit_range: + return limit + + result = min([x for x in limit_range if limit <= x] + [max(limit_range)]) + if not range_required and limit > result: + # Range is not required - we can use None as parameter. + return None + return result + + @retrier + def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: + """ + Get L2 order book from exchange. + Can be limited to a certain amount (if supported). + Returns a dict in the format + {'asks': [price, volume], 'bids': [price, volume]} + """ + limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'], + self._ft_has['l2_limit_range_required']) + try: + + return self._api.fetch_l2_order_book(pair, limit1) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching order book.' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1, + order_book_min: int = 1): + """ + Helper generator to query orderbook in loop (used for early sell-order placing) + """ + order_book = self.fetch_l2_order_book(pair, order_book_max) + for i in range(order_book_min, order_book_max + 1): + yield order_book[side][i - 1][0] + + def get_buy_rate(self, pair: str, refresh: bool) -> float: + """ + Calculates bid target between current ask price and last price + :param pair: Pair to get rate for + :param refresh: allow cached data + :return: float: Price + :raises PricingError if orderbook price could not be determined. + """ + if not refresh: + rate = self._buy_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.debug(f"Using cached buy rate for {pair}.") + return rate + + bid_strategy = self._config.get('bid_strategy', {}) + if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): + + order_book_top = bid_strategy.get('order_book_top', 1) + order_book = self.fetch_l2_order_book(pair, order_book_top) + logger.debug('order_book %s', order_book) + # top 1 = index 0 + try: + rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] + except (IndexError, KeyError) as e: + logger.warning( + "Buy Price from orderbook could not be determined." + f"Orderbook: {order_book}" + ) + raise PricingError from e + logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side " + f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}") + used_rate = rate_from_l2 + else: + logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") + ticker = self.fetch_ticker(pair) + ticker_rate = ticker[bid_strategy['price_side']] + if ticker['last'] and ticker_rate > ticker['last']: + balance = bid_strategy['ask_last_balance'] + ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) + used_rate = ticker_rate + + self._buy_rate_cache[pair] = used_rate + + return used_rate + + def get_sell_rate(self, pair: str, refresh: bool) -> float: + """ + Get sell rate - either using ticker bid or first bid based on orderbook + or remain static in any other case since it's not updating. + :param pair: Pair to get rate for + :param refresh: allow cached data + :return: Bid rate + :raises PricingError if price could not be determined. + """ + if not refresh: + rate = self._sell_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.debug(f"Using cached sell rate for {pair}.") + return rate + + ask_strategy = self._config.get('ask_strategy', {}) + if ask_strategy.get('use_order_book', False): + # This code is only used for notifications, selling uses the generator directly + logger.info( + f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." + ) + try: + rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) + except (IndexError, KeyError) as e: + logger.warning("Sell Price at location from orderbook could not be determined.") + raise PricingError from e + else: + ticker = self.fetch_ticker(pair) + ticker_rate = ticker[ask_strategy['price_side']] + if ticker['last'] and ticker_rate < ticker['last']: + balance = ask_strategy.get('bid_last_balance', 0.0) + ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last']) + rate = ticker_rate + + if rate is None: + raise PricingError(f"Sell-Rate for {pair} was empty.") + self._sell_rate_cache[pair] = rate + return rate + + # Fee handling + + @retrier + def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: + """ + Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. + The "since" argument passed in is coming from the database and is in UTC, + as timezone-native datetime object. + From the python documentation: + > Naive datetime instances are assumed to represent local time + Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the + transformation from local timezone to UTC. + This works for timezones UTC+ since then the result will contain trades from a few hours + instead of from the last 5 seconds, however fails for UTC- timezones, + since we're then asking for trades with a "since" argument in the future. + + :param order_id order_id: Order-id as given when creating the order + :param pair: Pair the order is for + :param since: datetime object of the order creation time. Assumes object is in UTC. + """ + if self._config['dry_run']: + return [] + if not self.exchange_has('fetchMyTrades'): + return [] + try: + # Allow 5s offset to catch slight time offsets (discovered in #1185) + # since needs to be int in milliseconds + my_trades = self._api.fetch_my_trades( + pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) + matched_trades = [trade for trade in my_trades if trade['order'] == order_id] + + self._log_exchange_response('get_trades_for_order', matched_trades) + return matched_trades + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + def get_order_id_conditional(self, order: Dict[str, Any]) -> str: + return order['id'] + + @retrier + def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, + price: float = 1, taker_or_maker: str = 'maker') -> float: + try: + if self._config['dry_run'] and self._config.get('fee', None) is not None: + return self._config['fee'] + # validate that markets are loaded before trying to get fee + if self._api.markets is None or len(self._api.markets) == 0: + self._api.load_markets() + + return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, + price=price, takerOrMaker=taker_or_maker)['rate'] + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @staticmethod + def order_has_fee(order: Dict) -> bool: + """ + Verifies if the passed in order dict has the needed keys to extract fees, + and that these keys (currency, cost) are not empty. + :param order: Order or trade (one trade) dict + :return: True if the fee substructure contains currency and cost, false otherwise + """ + if not isinstance(order, dict): + return False + return ('fee' in order and order['fee'] is not None + and (order['fee'].keys() >= {'currency', 'cost'}) + and order['fee']['currency'] is not None + and order['fee']['cost'] is not None + ) + + def calculate_fee_rate(self, order: Dict) -> Optional[float]: + """ + Calculate fee rate if it's not given by the exchange. + :param order: Order or trade (one trade) dict + """ + if order['fee'].get('rate') is not None: + return order['fee'].get('rate') + fee_curr = order['fee']['currency'] + # Calculate fee based on order details + if fee_curr in self.get_pair_base_currency(order['symbol']): + # Base currency - divide by amount + return round( + order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8) + elif fee_curr in self.get_pair_quote_currency(order['symbol']): + # Quote currency - divide by cost + return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None + else: + # If Fee currency is a different currency + if not order['cost']: + # If cost is None or 0.0 -> falsy, return None + return None + try: + comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) + tick = self.fetch_ticker(comb) + + fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') + return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) + except ExchangeError: + return None + + def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: + """ + Extract tuple of cost, currency, rate. + Requires order_has_fee to run first! + :param order: Order or trade (one trade) dict + :return: Tuple with cost, currency, rate of the given fee dict + """ + return (order['fee']['cost'], + order['fee']['currency'], + self.calculate_fee_rate(order)) + + # Historic data + def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: """ @@ -896,6 +1384,8 @@ class Exchange: raise OperationalException(f'Could not fetch historical candle (OHLCV) data ' f'for pair {pair}. Message: {e}') from e + # Fetch historic trades + @retrier_async async def _async_fetch_trades(self, pair: str, since: Optional[int] = None, @@ -1054,292 +1544,6 @@ class Exchange: self._async_get_trade_history(pair=pair, since=since, until=until, from_id=from_id)) - def check_order_canceled_empty(self, order: Dict) -> bool: - """ - Verify if an order has been cancelled without being partially filled - :param order: Order dict as returned from fetch_order() - :return: True if order has been cancelled without being filled, False otherwise. - """ - return (order.get('status') in ('closed', 'canceled', 'cancelled') - and order.get('filled') == 0.0) - - @retrier - def cancel_order(self, order_id: str, pair: str) -> Dict: - if self._config['dry_run']: - order = self._dry_run_open_orders.get(order_id) - if order: - order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) - return order - else: - return {} - - try: - return self._api.cancel_order(order_id, pair) - except ccxt.InvalidOrder as e: - raise InvalidOrderException( - f'Could not cancel order. Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - # Assign method to cancel_stoploss_order to allow easy overriding in other classes - cancel_stoploss_order = cancel_order - - def is_cancel_order_result_suitable(self, corder) -> bool: - if not isinstance(corder, dict): - return False - - required = ('fee', 'status', 'amount') - return all(k in corder for k in required) - - def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: - """ - Cancel order returning a result. - Creates a fake result if cancel order returns a non-usable result - and fetch_order does not work (certain exchanges don't return cancelled orders) - :param order_id: Orderid to cancel - :param pair: Pair corresponding to order_id - :param amount: Amount to use for fake response - :return: Result from either cancel_order if usable, or fetch_order - """ - try: - corder = self.cancel_order(order_id, pair) - if self.is_cancel_order_result_suitable(corder): - return corder - except InvalidOrderException: - logger.warning(f"Could not cancel order {order_id} for {pair}.") - try: - order = self.fetch_order(order_id, pair) - except InvalidOrderException: - logger.warning(f"Could not fetch cancelled order {order_id}.") - order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} - - return order - - def cancel_stoploss_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: - """ - Cancel stoploss order returning a result. - Creates a fake result if cancel order returns a non-usable result - and fetch_order does not work (certain exchanges don't return cancelled orders) - :param order_id: stoploss-order-id to cancel - :param pair: Pair corresponding to order_id - :param amount: Amount to use for fake response - :return: Result from either cancel_order if usable, or fetch_order - """ - corder = self.cancel_stoploss_order(order_id, pair) - if self.is_cancel_order_result_suitable(corder): - return corder - try: - order = self.fetch_stoploss_order(order_id, pair) - except InvalidOrderException: - logger.warning(f"Could not fetch cancelled stoploss order {order_id}.") - order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} - - return order - - @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) - def fetch_order(self, order_id: str, pair: str) -> Dict: - if self._config['dry_run']: - try: - order = self._dry_run_open_orders[order_id] - return order - except KeyError as e: - # Gracefully handle errors with dry-run orders. - raise InvalidOrderException( - f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e - try: - return self._api.fetch_order(order_id, pair) - except ccxt.OrderNotFound as e: - raise RetryableOrderError( - f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e - except ccxt.InvalidOrder as e: - raise InvalidOrderException( - f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - # Assign method to fetch_stoploss_order to allow easy overriding in other classes - fetch_stoploss_order = fetch_order - - def fetch_order_or_stoploss_order(self, order_id: str, pair: str, - stoploss_order: bool = False) -> Dict: - """ - Simple wrapper calling either fetch_order or fetch_stoploss_order depending on - the stoploss_order parameter - :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. - """ - if stoploss_order: - return self.fetch_stoploss_order(order_id, pair) - return self.fetch_order(order_id, pair) - - @staticmethod - def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]], - range_required: bool = True): - """ - Get next greater value in the list. - Used by fetch_l2_order_book if the api only supports a limited range - """ - if not limit_range: - return limit - - result = min([x for x in limit_range if limit <= x] + [max(limit_range)]) - if not range_required and limit > result: - # Range is not required - we can use None as parameter. - return None - return result - - @retrier - def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: - """ - Get L2 order book from exchange. - Can be limited to a certain amount (if supported). - Returns a dict in the format - {'asks': [price, volume], 'bids': [price, volume]} - """ - limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'], - self._ft_has['l2_limit_range_required']) - try: - - return self._api.fetch_l2_order_book(pair, limit1) - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching order book.' - f'Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - @retrier - def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: - """ - Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. - The "since" argument passed in is coming from the database and is in UTC, - as timezone-native datetime object. - From the python documentation: - > Naive datetime instances are assumed to represent local time - Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the - transformation from local timezone to UTC. - This works for timezones UTC+ since then the result will contain trades from a few hours - instead of from the last 5 seconds, however fails for UTC- timezones, - since we're then asking for trades with a "since" argument in the future. - - :param order_id order_id: Order-id as given when creating the order - :param pair: Pair the order is for - :param since: datetime object of the order creation time. Assumes object is in UTC. - """ - if self._config['dry_run']: - return [] - if not self.exchange_has('fetchMyTrades'): - return [] - try: - # Allow 5s offset to catch slight time offsets (discovered in #1185) - # since needs to be int in milliseconds - my_trades = self._api.fetch_my_trades( - pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) - matched_trades = [trade for trade in my_trades if trade['order'] == order_id] - - return matched_trades - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - def get_order_id_conditional(self, order: Dict[str, Any]) -> str: - return order['id'] - - @retrier - def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, - price: float = 1, taker_or_maker: str = 'maker') -> float: - try: - if self._config['dry_run'] and self._config.get('fee', None) is not None: - return self._config['fee'] - # validate that markets are loaded before trying to get fee - if self._api.markets is None or len(self._api.markets) == 0: - self._api.load_markets() - - return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, - price=price, takerOrMaker=taker_or_maker)['rate'] - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - @staticmethod - def order_has_fee(order: Dict) -> bool: - """ - Verifies if the passed in order dict has the needed keys to extract fees, - and that these keys (currency, cost) are not empty. - :param order: Order or trade (one trade) dict - :return: True if the fee substructure contains currency and cost, false otherwise - """ - if not isinstance(order, dict): - return False - return ('fee' in order and order['fee'] is not None - and (order['fee'].keys() >= {'currency', 'cost'}) - and order['fee']['currency'] is not None - and order['fee']['cost'] is not None - ) - - def calculate_fee_rate(self, order: Dict) -> Optional[float]: - """ - Calculate fee rate if it's not given by the exchange. - :param order: Order or trade (one trade) dict - """ - if order['fee'].get('rate') is not None: - return order['fee'].get('rate') - fee_curr = order['fee']['currency'] - # Calculate fee based on order details - if fee_curr in self.get_pair_base_currency(order['symbol']): - # Base currency - divide by amount - return round( - order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8) - elif fee_curr in self.get_pair_quote_currency(order['symbol']): - # Quote currency - divide by cost - return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None - else: - # If Fee currency is a different currency - if not order['cost']: - # If cost is None or 0.0 -> falsy, return None - return None - try: - comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) - tick = self.fetch_ticker(comb) - - fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') - return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) - except ExchangeError: - return None - - def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: - """ - Extract tuple of cost, currency, rate. - Requires order_has_fee to run first! - :param order: Order or trade (one trade) dict - :return: Tuple with cost, currency, rate of the given fee dict - """ - return (order['fee']['cost'], - order['fee']['currency'], - self.calculate_fee_rate(order)) - def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool: return exchange_name in ccxt_exchanges(ccxt_module) diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index 9009e9492..6cd549d60 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -69,6 +69,7 @@ class Ftx(Exchange): order = self._api.create_order(symbol=pair, type=ordertype, side='sell', amount=amount, params=params) + self._log_exchange_response('create_stoploss_order', order) logger.info('stoploss order added for %s. ' 'stop price: %s.', pair, stop_price) return order @@ -93,18 +94,26 @@ class Ftx(Exchange): @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - try: - order = self._dry_run_open_orders[order_id] - return order - except KeyError as e: - # Gracefully handle errors with dry-run orders. - raise InvalidOrderException( - f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + return self.fetch_dry_run_order(order_id) + try: orders = self._api.fetch_orders(pair, None, params={'type': 'stop'}) order = [order for order in orders if order['id'] == order_id] + self._log_exchange_response('fetch_stoploss_order', order) if len(order) == 1: + if order[0].get('status') == 'closed': + # Trigger order was triggered ... + real_order_id = order[0].get('info', {}).get('orderId') + + order1 = self._api.fetch_order(real_order_id, pair) + self._log_exchange_response('fetch_stoploss_order1', order1) + # Fake type to stop - as this was really a stop order. + order1['id_stop'] = order1['id'] + order1['id'] = order_id + order1['type'] = 'stop' + order1['status_stop'] = 'triggered' + return order1 return order[0] else: raise InvalidOrderException(f"Could not get stoploss order for id {order_id}") @@ -125,7 +134,9 @@ class Ftx(Exchange): if self._config['dry_run']: return {} try: - return self._api.cancel_order(order_id, pair, params={'type': 'stop'}) + order = self._api.cancel_order(order_id, pair, params={'type': 'stop'}) + self._log_exchange_response('cancel_stoploss_order', order) + return order except ccxt.InvalidOrder as e: raise InvalidOrderException( f'Could not cancel order. Message: {e}') from e @@ -139,5 +150,5 @@ class Ftx(Exchange): def get_order_id_conditional(self, order: Dict[str, Any]) -> str: if order['type'] == 'stop': - return safe_value_fallback2(order['info'], order, 'orderId', 'id') + return safe_value_fallback2(order, order, 'id_stop', 'id') return order['id'] diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 6f1fa409a..1b069aa6c 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -49,7 +49,7 @@ class Kraken(Exchange): orders = self._api.fetch_open_orders() order_list = [(x["symbol"].split("/")[0 if x["side"] == "sell" else 1], x["remaining"] if x["side"] == "sell" else x["remaining"] * x["price"], - # Don't remove the below comment, this can be important for debuggung + # Don't remove the below comment, this can be important for debugging # x["side"], x["amount"], ) for x in orders] for bal in balances: @@ -103,6 +103,7 @@ class Kraken(Exchange): order = self._api.create_order(symbol=pair, type=ordertype, side='sell', amount=amount, price=stop_price, params=params) + self._log_exchange_response('create_stoploss_order', order) logger.info('stoploss order added for %s. ' 'stop price: %s.', pair, stop_price) return order diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 56596f2b9..e533e15e0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -10,13 +10,13 @@ from threading import Lock from typing import Any, Dict, List, Optional import arrow -from cachetools import TTLCache from freqtrade import __version__, constants from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge +from freqtrade.enums import RPCMessageType, SellType, State from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds @@ -26,9 +26,8 @@ from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.state import State -from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType +from freqtrade.rpc import RPCManager +from freqtrade.strategy.interface import IStrategy, SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -48,6 +47,7 @@ class FreqtradeBot(LoggingMixin): :param config: configuration dict, you can use Configuration.get_config() to get the config dict. """ + self.active_pair_whitelist: List[str] = [] logger.info('Starting freqtrade %s', __version__) @@ -57,12 +57,6 @@ class FreqtradeBot(LoggingMixin): # Init objects self.config = config - # Cache values for 1800 to avoid frequent polling of the exchange for prices - # Caching only applies to RPC methods, so prices for open trades are still - # refreshed once every iteration. - self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) - self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) - self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) # Check config consistency here since strategies can set certain options @@ -76,12 +70,19 @@ class FreqtradeBot(LoggingMixin): PairLocks.timeframe = self.config['timeframe'] + self.protections = ProtectionManager(self.config, self.strategy.protections) + + # RPC runs in separate threads, can start handling external commands just after + # initialization, even before Freqtradebot has a chance to start its throttling, + # so anything in the Freqtradebot instance should be ready (initialized), including + # the initial state of the bot. + # Keep this at the end of this initialization method. + self.rpc: RPCManager = RPCManager(self) + self.pairlists = PairListManager(self.exchange, self.config) self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) - self.protections = ProtectionManager(self.config) - # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass @@ -97,13 +98,7 @@ class FreqtradeBot(LoggingMixin): initial_state = self.config.get('initial_state') self.state = State[initial_state.upper()] if initial_state else State.STOPPED - # RPC runs in separate threads, can start handling external commands just after - # initialization, even before Freqtradebot has a chance to start its throttling, - # so anything in the Freqtradebot instance should be ready (initialized), including - # the initial state of the bot. - # Keep this at the end of this initialization method. - self.rpc: RPCManager = RPCManager(self) - # Protect sell-logic from forcesell and viceversa + # Protect sell-logic from forcesell and vice versa self._sell_lock = Lock() LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) @@ -187,7 +182,7 @@ class FreqtradeBot(LoggingMixin): if self.get_free_open_trades(): self.enter_positions() - Trade.query.session.flush() + Trade.commit() def process_stopped(self) -> None: """ @@ -342,7 +337,7 @@ class FreqtradeBot(LoggingMixin): # Assume this as the open order trade.open_order_id = order.order_id if fo: - logger.info(f"Found {order} for trade {trade}.jj") + logger.info(f"Found {order} for trade {trade}.") self.update_trade_state(trade, order.order_id, fo, stoploss_order=order.ft_order_side == 'stoploss') @@ -394,51 +389,6 @@ class FreqtradeBot(LoggingMixin): return trades_created - def get_buy_rate(self, pair: str, refresh: bool) -> float: - """ - Calculates bid target between current ask price and last price - :param pair: Pair to get rate for - :param refresh: allow cached data - :return: float: Price - """ - if not refresh: - rate = self._buy_rate_cache.get(pair) - # Check if cache has been invalidated - if rate: - logger.debug(f"Using cached buy rate for {pair}.") - return rate - - bid_strategy = self.config.get('bid_strategy', {}) - if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): - - order_book_top = bid_strategy.get('order_book_top', 1) - order_book = self.exchange.fetch_l2_order_book(pair, order_book_top) - logger.debug('order_book %s', order_book) - # top 1 = index 0 - try: - rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] - except (IndexError, KeyError) as e: - logger.warning( - "Buy Price from orderbook could not be determined." - f"Orderbook: {order_book}" - ) - raise PricingError from e - logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side " - f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}") - used_rate = rate_from_l2 - else: - logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") - ticker = self.exchange.fetch_ticker(pair) - ticker_rate = ticker[bid_strategy['price_side']] - if ticker['last'] and ticker_rate > ticker['last']: - balance = bid_strategy['ask_last_balance'] - ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) - used_rate = ticker_rate - - self._buy_rate_cache[pair] = used_rate - - return used_rate - def create_trade(self, pair: str) -> bool: """ Check the implemented trading strategy for buy signals. @@ -522,6 +472,7 @@ class FreqtradeBot(LoggingMixin): """ Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY + :param stake_amount: amount of stake-currency for the pair :return: True if a buy order is created, false if it fails. """ time_in_force = self.strategy.order_time_in_force['buy'] @@ -530,7 +481,7 @@ class FreqtradeBot(LoggingMixin): buy_limit_requested = price else: # Calculate price - buy_limit_requested = self.get_buy_rate(pair, True) + buy_limit_requested = self.exchange.get_buy_rate(pair, True) if not buy_limit_requested: raise PricingError('Could not determine buy price.') @@ -620,7 +571,7 @@ class FreqtradeBot(LoggingMixin): self.update_trade_state(trade, order_id, order) Trade.query.session.add(trade) - Trade.query.session.flush() + Trade.commit() # Updating wallets self.wallets.update() @@ -655,7 +606,7 @@ class FreqtradeBot(LoggingMixin): """ Sends rpc notification when a buy cancel occurred. """ - current_rate = self.get_buy_rate(trade.pair, False) + current_rate = self.exchange.get_buy_rate(trade.pair, False) msg = { 'trade_id': trade.id, @@ -706,6 +657,7 @@ class FreqtradeBot(LoggingMixin): if (self.strategy.order_types.get('stoploss_on_exchange') and self.handle_stoploss_on_exchange(trade)): trades_closed += 1 + Trade.commit() continue # Check if we can sell our current pair if trade.open_order_id is None and trade.is_open and self.handle_trade(trade): @@ -720,56 +672,6 @@ class FreqtradeBot(LoggingMixin): return trades_closed - def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1, - order_book_min: int = 1): - """ - Helper generator to query orderbook in loop (used for early sell-order placing) - """ - order_book = self.exchange.fetch_l2_order_book(pair, order_book_max) - for i in range(order_book_min, order_book_max + 1): - yield order_book[side][i - 1][0] - - def get_sell_rate(self, pair: str, refresh: bool) -> float: - """ - Get sell rate - either using ticker bid or first bid based on orderbook - The orderbook portion is only used for rpc messaging, which would otherwise fail - for BitMex (has no bid/ask in fetch_ticker) - or remain static in any other case since it's not updating. - :param pair: Pair to get rate for - :param refresh: allow cached data - :return: Bid rate - """ - if not refresh: - rate = self._sell_rate_cache.get(pair) - # Check if cache has been invalidated - if rate: - logger.debug(f"Using cached sell rate for {pair}.") - return rate - - ask_strategy = self.config.get('ask_strategy', {}) - if ask_strategy.get('use_order_book', False): - # This code is only used for notifications, selling uses the generator directly - logger.info( - f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." - ) - try: - rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) - except (IndexError, KeyError) as e: - logger.warning("Sell Price at location from orderbook could not be determined.") - raise PricingError from e - else: - ticker = self.exchange.fetch_ticker(pair) - ticker_rate = ticker[ask_strategy['price_side']] - if ticker['last'] and ticker_rate < ticker['last']: - balance = ask_strategy.get('bid_last_balance', 0.0) - ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last']) - rate = ticker_rate - - if rate is None: - raise PricingError(f"Sell-Rate for {pair} was empty.") - self._sell_rate_cache[pair] = rate - return rate - def handle_trade(self, trade: Trade) -> bool: """ Sells the current pair if the threshold is reached and updates the trade record. @@ -797,9 +699,9 @@ class FreqtradeBot(LoggingMixin): logger.debug(f'Using order book between {order_book_min} and {order_book_max} ' f'for selling {trade.pair}...') - order_book = self._order_book_gen(trade.pair, f"{config_ask_strategy['price_side']}s", - order_book_min=order_book_min, - order_book_max=order_book_max) + order_book = self.exchange._order_book_gen( + trade.pair, f"{config_ask_strategy['price_side']}s", + order_book_min=order_book_min, order_book_max=order_book_max) for i in range(order_book_min, order_book_max + 1): try: sell_rate = next(order_book) @@ -812,14 +714,14 @@ class FreqtradeBot(LoggingMixin): f"{sell_rate:0.8f}") # Assign sell-rate to cache - otherwise sell-rate is never updated in the cache, # resulting in outdated RPC messages - self._sell_rate_cache[trade.pair] = sell_rate + self.exchange._sell_rate_cache[trade.pair] = sell_rate if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True else: logger.debug('checking sell') - sell_rate = self.get_sell_rate(trade.pair, True) + sell_rate = self.exchange.get_sell_rate(trade.pair, True) if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True @@ -915,8 +817,13 @@ class FreqtradeBot(LoggingMixin): logger.warning('Stoploss order was cancelled, but unable to recreate one.') # Finally we check if stoploss on exchange should be moved up because of trailing. - if stoploss_order and (self.config.get('trailing_stop', False) - or self.config.get('use_custom_stoploss', False)): + # Triggered Orders are now real orders - so don't replace stoploss anymore + if ( + stoploss_order + and stoploss_order.get('status_stop') != 'triggered' + and (self.config.get('trailing_stop', False) + or self.config.get('use_custom_stoploss', False)) + ): # if trailing stoploss is enabled we check if stoploss value has changed # in which case we cancel stoploss order and put another one with new # value immediately @@ -928,7 +835,7 @@ class FreqtradeBot(LoggingMixin): """ Check to see if stoploss on exchange should be updated in case of trailing stoploss on exchange - :param Trade: Corresponding Trade + :param trade: Corresponding Trade :param order: Current on exchange stoploss order :return: None """ @@ -1036,6 +943,7 @@ class FreqtradeBot(LoggingMixin): elif order['side'] == 'sell': self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + Trade.commit() def handle_cancel_buy(self, trade: Trade, order: Dict, reason: str) -> bool: """ @@ -1233,7 +1141,7 @@ class FreqtradeBot(LoggingMixin): # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') == 'closed': self.update_trade_state(trade, trade.open_order_id, order) - Trade.query.session.flush() + Trade.commit() # Lock pair for one candle to prevent immediate re-buys self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), @@ -1250,7 +1158,7 @@ class FreqtradeBot(LoggingMixin): profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) # Use cached rates here - it was updated seconds ago. - current_rate = self.get_sell_rate(trade.pair, False) if not fill else None + current_rate = self.exchange.get_sell_rate(trade.pair, False) if not fill else None profit_ratio = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_ratio > 0 else "loss" @@ -1295,7 +1203,7 @@ class FreqtradeBot(LoggingMixin): profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) - current_rate = self.get_sell_rate(trade.pair, False) + current_rate = self.exchange.get_sell_rate(trade.pair, False) profit_ratio = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_ratio > 0 else "loss" @@ -1374,6 +1282,7 @@ class FreqtradeBot(LoggingMixin): # Handling of this will happen in check_handle_timeout. return True trade.update(order) + Trade.commit() # Updating wallets when order is closed if not trade.is_open: diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 2e255901e..967f08299 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -56,6 +56,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = """ Dump JSON data into a file :param filename: file to create + :param is_zip: if file should be zip :param data: JSON Data to save :return: """ diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cbc0995aa..e2ca39530 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -17,6 +17,7 @@ from freqtrade.data import history from freqtrade.data.btanalysis import trade_list_to_dataframe from freqtrade.data.converter import trim_dataframes from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums import SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.mixins import LoggingMixin @@ -26,7 +27,7 @@ from freqtrade.persistence import LocalTrade, PairLocks, Trade from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType +from freqtrade.strategy.interface import IStrategy, SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -136,7 +137,7 @@ class Backtesting: if hasattr(strategy, 'protections'): conf = deepcopy(conf) conf['protections'] = strategy.protections - self.protections = ProtectionManager(conf) + self.protections = ProtectionManager(self.config, strategy.protections) def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: """ @@ -224,6 +225,22 @@ class Backtesting: # sell at open price. return sell_row[OPEN_IDX] + # Special case: trailing triggers within same candle as trade opened. Assume most + # pessimistic price movement, which is moving just enough to arm stoploss and + # immediately going down to stop price. + if (sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0 + and self.strategy.trailing_stop_positive): + if self.strategy.trailing_only_offset_is_reached: + # Worst case: price reaches stop_positive_offset and dives down. + stop_rate = (sell_row[OPEN_IDX] * + (1 + abs(self.strategy.trailing_stop_positive_offset) - + abs(self.strategy.trailing_stop_positive))) + else: + # Worst case: price ticks tiny bit above open and dives down. + stop_rate = sell_row[OPEN_IDX] * (1 - abs(self.strategy.trailing_stop_positive)) + assert stop_rate < sell_row[HIGH_IDX] + return stop_rate + # Set close_rate to stoploss return trade.stop_loss elif sell.sell_type == (SellType.ROI): @@ -428,7 +445,7 @@ class Backtesting: for trade in open_trades[pair]: # also check the buying candle for sell conditions. trade_entry = self._get_sell_trade_entry(trade, row) - # Sell occured + # Sell occurred if trade_entry: # logger.debug(f"{pair} - Backtesting sell {trade}") open_trade_count -= 1 @@ -519,7 +536,7 @@ class Backtesting: stats = generate_backtest_stats(data, self.all_results, min_date=min_date, max_date=max_date) - if self.config.get('export', False): + if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], stats) # Show backtest results diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 49273d3de..c2b2b93cb 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -469,8 +469,8 @@ class Hyperopt: f"saved to '{self.results_file}'.") if self.current_best_epoch: - HyperoptTools.print_epoch_details(self.current_best_epoch, self.total_epochs, - self.print_json) + HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs, + self.print_json) else: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. diff --git a/freqtrade/optimize/hyperopt_loss_onlyprofit.py b/freqtrade/optimize/hyperopt_loss_onlyprofit.py index 33f3f5bc6..4a3cf1b3b 100644 --- a/freqtrade/optimize/hyperopt_loss_onlyprofit.py +++ b/freqtrade/optimize/hyperopt_loss_onlyprofit.py @@ -9,23 +9,11 @@ from pandas import DataFrame from freqtrade.optimize.hyperopt import IHyperOptLoss -# This is assumed to be expected avg profit * expected trade count. -# For example, for 0.35% avg per trade (or 0.0035 as ratio) and 1100 trades, -# expected max profit = 3.85 -# -# Note, this is ratio. 3.85 stated above means 385Σ%, 3.0 means 300Σ%. -# -# In this implementation it's only used in calculation of the resulting value -# of the objective function as a normalization coefficient and does not -# represent any limit for profits as in the Freqtrade legacy default loss function. -EXPECTED_MAX_PROFIT = 3.0 - - class OnlyProfitHyperOptLoss(IHyperOptLoss): """ Defines the loss function for hyperopt. - This implementation takes only profit into account. + This implementation takes only absolute profit into account, not looking at any other indicator. """ @staticmethod @@ -34,5 +22,5 @@ class OnlyProfitHyperOptLoss(IHyperOptLoss): """ Objective function, returns smaller number for better results. """ - total_profit = results['profit_ratio'].sum() - return 1 - total_profit / EXPECTED_MAX_PROFIT + total_profit = results['profit_abs'].sum() + return -1 * total_profit diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py old mode 100644 new mode 100755 index 38cb0854e..9eee42a8d --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -1,8 +1,6 @@ import io -import locale import logging -from collections import OrderedDict from pathlib import Path from typing import Any, Dict, List @@ -74,8 +72,8 @@ class HyperoptTools(): return epochs @staticmethod - def print_epoch_details(results, total_epochs: int, print_json: bool, - no_header: bool = False, header_str: str = None) -> None: + def show_epoch_details(results, total_epochs: int, print_json: bool, + no_header: bool = False, header_str: str = None) -> None: """ Display details of the hyperopt result """ @@ -93,7 +91,7 @@ class HyperoptTools(): if print_json: result_dict: Dict = {} for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: - HyperoptTools._params_update_for_json(result_dict, params, s) + HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s) print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE)) else: @@ -106,24 +104,24 @@ class HyperoptTools(): HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:") @staticmethod - def _params_update_for_json(result_dict, params, space: str) -> None: - if space in params: + def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None: + if (space in params) or (space in non_optimized): space_params = HyperoptTools._space_params(params, space) - if space in ['buy', 'sell']: - result_dict.setdefault('params', {}).update(space_params) - elif space == 'roi': - # TODO: get rid of OrderedDict when support for python 3.6 will be - # dropped (dicts keep the order as the language feature) + space_non_optimized = HyperoptTools._space_params(non_optimized, space) + all_space_params = space_params + # Merge non optimized params if there are any + if len(space_non_optimized) > 0: + all_space_params = {**space_params, **space_non_optimized} + + if space in ['buy', 'sell']: + result_dict.setdefault('params', {}).update(all_space_params) + elif space == 'roi': # Convert keys in min_roi dict to strings because # rapidjson cannot dump dicts with integer keys... - # OrderedDict is used to keep the numeric order of the items - # in the dict. - result_dict['minimal_roi'] = OrderedDict( - (str(k), v) for k, v in space_params.items() - ) + result_dict['minimal_roi'] = {str(k): v for k, v in all_space_params.items()} else: # 'stoploss', 'trailing' - result_dict.update(space_params) + result_dict.update(all_space_params) @staticmethod def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None: @@ -133,13 +131,9 @@ class HyperoptTools(): if space == 'stoploss': result += f"stoploss = {space_params.get('stoploss')}" elif space == 'roi': - # TODO: get rid of OrderedDict when support for python 3.6 will be - # dropped (dicts keep the order as the language feature) - minimal_roi_result = rapidjson.dumps( - OrderedDict( - (str(k), v) for k, v in space_params.items() - ), - default=str, indent=4, number_mode=rapidjson.NM_NATIVE) + minimal_roi_result = rapidjson.dumps({ + str(k): v for k, v in space_params.items() + }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE) result += f"minimal_roi = {minimal_roi_result}" elif space == 'trailing': @@ -195,9 +189,9 @@ class HyperoptTools(): f"Avg profit {results_metrics['profit_mean'] * 100: 6.2f}%. " f"Median profit {results_metrics['profit_median'] * 100: 6.2f}%. " f"Total profit {results_metrics['profit_total_abs']: 11.8f} {stake_currency} " - f"({results_metrics['profit_total'] * 100: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). " + f"({results_metrics['profit_total'] * 100: 7.2f}%). " f"Avg duration {results_metrics['holding_avg']} min." - ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8') + ) @staticmethod def _format_explanation_string(results, total_epochs) -> str: @@ -206,6 +200,47 @@ class HyperoptTools(): f"{results['results_explanation']} " + f"Objective: {results['loss']:.5f}") + @staticmethod + def prepare_trials_columns(trials, legacy_mode: bool, has_drawdown: bool) -> str: + + trials['Best'] = '' + + if 'results_metrics.winsdrawslosses' not in trials.columns: + # Ensure compatibility with older versions of hyperopt results + trials['results_metrics.winsdrawslosses'] = 'N/A' + + if not has_drawdown: + # Ensure compatibility with older versions of hyperopt results + trials['results_metrics.max_drawdown_abs'] = None + trials['results_metrics.max_drawdown'] = None + + if not legacy_mode: + # New mode, using backtest result for metrics + trials['results_metrics.winsdrawslosses'] = trials.apply( + lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} " + f"{x['results_metrics.losses']:>4}", axis=1) + trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades', + 'results_metrics.winsdrawslosses', + 'results_metrics.profit_mean', 'results_metrics.profit_total_abs', + 'results_metrics.profit_total', 'results_metrics.holding_avg', + 'results_metrics.max_drawdown', 'results_metrics.max_drawdown_abs', + 'loss', 'is_initial_point', 'is_best']] + + else: + # Legacy mode + trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', + 'results_metrics.winsdrawslosses', 'results_metrics.avg_profit', + 'results_metrics.total_profit', 'results_metrics.profit', + 'results_metrics.duration', 'results_metrics.max_drawdown', + 'results_metrics.max_drawdown_abs', 'loss', 'is_initial_point', + 'is_best']] + + trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', + 'Total profit', 'Profit', 'Avg duration', 'Max Drawdown', + 'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best'] + + return trials + @staticmethod def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, print_colorized: bool, remove_header: int) -> str: @@ -216,36 +251,13 @@ class HyperoptTools(): return '' tabulate.PRESERVE_WHITESPACE = True - trials = json_normalize(results, max_level=1) - trials['Best'] = '' - if 'results_metrics.winsdrawslosses' not in trials.columns: - # Ensure compatibility with older versions of hyperopt results - trials['results_metrics.winsdrawslosses'] = 'N/A' - legacy_mode = True - if 'results_metrics.total_trades' in trials: - legacy_mode = False - # New mode, using backtest result for metrics - trials['results_metrics.winsdrawslosses'] = trials.apply( - lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} " - f"{x['results_metrics.losses']:>4}", axis=1) - trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades', - 'results_metrics.winsdrawslosses', - 'results_metrics.profit_mean', 'results_metrics.profit_total_abs', - 'results_metrics.profit_total', 'results_metrics.holding_avg', - 'loss', 'is_initial_point', 'is_best']] - else: - # Legacy mode - trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', - 'results_metrics.winsdrawslosses', - 'results_metrics.avg_profit', 'results_metrics.total_profit', - 'results_metrics.profit', 'results_metrics.duration', - 'loss', 'is_initial_point', 'is_best']] + legacy_mode = 'results_metrics.total_trades' not in trials + has_drawdown = 'results_metrics.max_drawdown_abs' in trials.columns + + trials = HyperoptTools.prepare_trials_columns(trials, legacy_mode, has_drawdown) - trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', - 'Total profit', 'Profit', 'Avg duration', 'Objective', - 'is_initial_point', 'is_best'] trials['is_profit'] = False trials.loc[trials['is_initial_point'], 'Best'] = '* ' trials.loc[trials['is_best'], 'Best'] = 'Best' @@ -268,6 +280,21 @@ class HyperoptTools(): ) stake_currency = config['stake_currency'] + + if has_drawdown: + trials['Max Drawdown'] = trials.apply( + lambda x: '{} {}'.format( + round_coin_value(x['max_drawdown_abs'], stake_currency), + '({:,.2f}%)'.format(x['Max Drawdown'] * perc_multi).rjust(10, ' ') + ).rjust(25 + len(stake_currency)) + if x['Max Drawdown'] != 0.0 else '--'.rjust(25 + len(stake_currency)), + axis=1 + ) + else: + trials = trials.drop(columns=['Max Drawdown']) + + trials = trials.drop(columns=['max_drawdown_abs']) + trials['Profit'] = trials.apply( lambda x: '{} {}'.format( round_coin_value(x['Total profit'], stake_currency), diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 5822fc627..24dbb4961 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -21,7 +21,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N Stores backtest results :param recordfilename: Path object, which can either be a filename or a directory. Filenames will be appended with a timestamp right before the suffix - while for diectories, /backtest-result-.json will be used as filename + while for directories, /backtest-result-.json will be used as filename :param stats: Dataframe containing the backtesting statistics """ if recordfilename.is_dir(): @@ -232,16 +232,23 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]: zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) & (results['sell_reason'] == 'trailing_stop_loss')]) + holding_avg = (timedelta(minutes=round(results['trade_duration'].mean())) + if not results.empty else timedelta()) + winner_holding_avg = (timedelta(minutes=round(winning_trades['trade_duration'].mean())) + if not winning_trades.empty else timedelta()) + loser_holding_avg = (timedelta(minutes=round(losing_trades['trade_duration'].mean())) + if not losing_trades.empty else timedelta()) + return { 'wins': len(winning_trades), 'losses': len(losing_trades), 'draws': len(draw_trades), - 'holding_avg': (timedelta(minutes=round(results['trade_duration'].mean())) - if not results.empty else timedelta()), - 'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean())) - if not winning_trades.empty else timedelta()), - 'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean())) - if not losing_trades.empty else timedelta()), + 'holding_avg': holding_avg, + 'holding_avg_s': holding_avg.total_seconds(), + 'winner_holding_avg': winner_holding_avg, + 'winner_holding_avg_s': winner_holding_avg.total_seconds(), + 'loser_holding_avg': loser_holding_avg, + 'loser_holding_avg_s': loser_holding_avg.total_seconds(), 'zero_duration_trades': zero_duration_trades, } @@ -293,7 +300,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame], :param min_date: Backtest start date :param max_date: Backtest end date :param market_change: float indicating the market change - :return: Dictionary containing results per strategy and a stratgy summary. + :return: Dictionary containing results per strategy and a strategy summary. """ results: Dict[str, DataFrame] = content['results'] if not isinstance(results, DataFrame): @@ -429,7 +436,7 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], { Strategy: {'results: results, 'config: config}}. :param min_date: Backtest start date :param max_date: Backtest end date - :return: Dictionary containing results per strategy and a stratgy summary. + :return: Dictionary containing results per strategy and a strategy summary. """ result: Dict[str, Any] = {'strategy': {}} market_change = calculate_market_change(btdata, 'close') @@ -500,9 +507,8 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren def text_table_strategy(strategy_results, stake_currency: str) -> str: """ Generate summary table per strategy + :param strategy_results: Dict of containing results for all strategies :param stake_currency: stake-currency - used to correctly name headers - :param max_open_trades: Maximum allowed open trades used for backtest - :param all_results: Dict of containing results for all strategies :return: pretty printed table with tabulate as string """ floatfmt = _get_line_floatfmt(stake_currency) @@ -549,7 +555,8 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Backtesting to', strat_results['backtest_end']), ('Max open trades', strat_results['max_open_trades']), ('', ''), # Empty line to improve readability - ('Total trades', strat_results['total_trades']), + ('Total/Daily Avg Trades', + f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"), ('Starting balance', round_coin_value(strat_results['starting_balance'], strat_results['stake_currency'])), ('Final balance', round_coin_value(strat_results['final_balance'], @@ -557,7 +564,6 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], strat_results['stake_currency'])), ('Total profit %', f"{round(strat_results['profit_total'] * 100, 2):}%"), - ('Trades per day', strat_results['trades_per_day']), ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'], strat_results['stake_currency'])), ('Total trade volume', round_coin_value(strat_results['total_volume'], diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index d89256baf..00c9b91eb 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -1,7 +1,7 @@ import logging from typing import List -from sqlalchemy import inspect +from sqlalchemy import inspect, text logger = logging.getLogger(__name__) @@ -62,15 +62,17 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col amount_requested = get_column_def(cols, 'amount_requested', 'amount') # Schema migration necessary - engine.execute(f"alter table trades rename to {table_back_name}") - # drop indexes on backup table - for index in inspector.get_indexes(table_back_name): - engine.execute(f"drop index {index['name']}") + with engine.begin() as connection: + connection.execute(text(f"alter table trades rename to {table_back_name}")) + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + connection.execute(text(f"drop index {index['name']}")) # let SQLAlchemy create the schema as required decl_base.metadata.create_all(engine) # Copy data back - following the correct schema - engine.execute(f"""insert into trades + with engine.begin() as connection: + connection.execute(text(f"""insert into trades (id, exchange, pair, is_open, fee_open, fee_open_cost, fee_open_currency, fee_close, fee_close_cost, fee_open_currency, open_rate, @@ -104,11 +106,12 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col {strategy} strategy, {timeframe} timeframe, {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs from {table_back_name} - """) + """)) def migrate_open_orders_to_trades(engine): - engine.execute(""" + with engine.begin() as connection: + connection.execute(text(""" insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open) select id ft_trade_id, pair ft_pair, open_order_id, case when close_rate_requested is null then 'buy' @@ -120,28 +123,30 @@ def migrate_open_orders_to_trades(engine): 'stoploss' ft_order_side, 1 ft_is_open from trades where stoploss_order_id is not null - """) + """)) def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, cols: List): # Schema migration necessary - engine.execute(f"alter table orders rename to {table_back_name}") - # drop indexes on backup table - for index in inspector.get_indexes(table_back_name): - engine.execute(f"drop index {index['name']}") + + with engine.begin() as connection: + connection.execute(text(f"alter table orders rename to {table_back_name}")) + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + connection.execute(text(f"drop index {index['name']}")) # let SQLAlchemy create the schema as required decl_base.metadata.create_all(engine) - - engine.execute(f""" - insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, - symbol, order_type, side, price, amount, filled, average, remaining, cost, order_date, - order_filled_date, order_update_date) - select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, - symbol, order_type, side, price, amount, filled, null average, remaining, cost, order_date, - order_filled_date, order_update_date - from {table_back_name} - """) + with engine.begin() as connection: + connection.execute(text(f""" + insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, + status, symbol, order_type, side, price, amount, filled, average, remaining, cost, + order_date, order_filled_date, order_update_date) + select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, + status, symbol, order_type, side, price, amount, filled, null average, remaining, cost, + order_date, order_filled_date, order_update_date + from {table_back_name} + """)) def check_migrate(engine, decl_base, previous_tables) -> None: diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index f2e7a10c4..bbbc55c13 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -9,14 +9,12 @@ from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Query, relationship -from sqlalchemy.orm.scoping import scoped_session -from sqlalchemy.orm.session import sessionmaker +from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker from sqlalchemy.pool import StaticPool from sqlalchemy.sql.schema import UniqueConstraint from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.enums import SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.misc import safe_value_fallback from freqtrade.persistence.migrations import check_migrate @@ -41,16 +39,18 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None: """ kwargs = {} - # Take care of thread ownership if in-memory db if db_url == 'sqlite://': kwargs.update({ - 'connect_args': {'check_same_thread': False}, 'poolclass': StaticPool, - 'echo': False, + }) + # Take care of thread ownership + if db_url.startswith('sqlite://'): + kwargs.update({ + 'connect_args': {'check_same_thread': False}, }) try: - engine = create_engine(db_url, **kwargs) + engine = create_engine(db_url, future=True, **kwargs) except NoSuchModuleError: raise OperationalException(f"Given value for db_url: '{db_url}' " f"is no valid database URL! (See {_SQL_DOCS_URL})") @@ -58,7 +58,7 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None: # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope # Scoped sessions proxy requests to the appropriate thread-local session. # We should use the scoped_session object - not a seperately initialized version - Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) + Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True)) Trade.query = Trade._session.query_property() Order.query = Trade._session.query_property() PairLock.query = Trade._session.query_property() @@ -77,7 +77,7 @@ def cleanup_db() -> None: Flushes all pending operations to disk. :return: None """ - Trade.query.session.flush() + Trade.commit() def clean_dry_run_db() -> None: @@ -89,6 +89,7 @@ def clean_dry_run_db() -> None: # Check we are updating only a dry_run order not a prod one if 'dry_run' in trade.open_order_id: trade.open_order_id = None + Trade.commit() class Order(_DECL_BASE): @@ -177,6 +178,7 @@ class Order(_DECL_BASE): if filtered_orders: oobj = filtered_orders[0] oobj.update_from_ccxt_object(order) + Order.query.session.commit() else: logger.warning(f"Did not find order for {order}.") @@ -429,12 +431,13 @@ class LocalTrade(): elif order_type in ('stop_loss_limit', 'stop-loss', 'stop-loss-limit', 'stop'): self.stoploss_order_id = None self.close_rate_requested = self.stop_loss + self.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value if self.is_open: logger.info(f'{order_type.upper()} is hit for {self}.') self.close(safe_value_fallback(order, 'average', 'price')) else: raise ValueError(f'Unknown order type: {order_type}') - cleanup_db() + Trade.commit() def close(self, rate: float, *, show_msg: bool = True) -> None: """ @@ -712,7 +715,11 @@ class Trade(_DECL_BASE, LocalTrade): Order.query.session.delete(order) Trade.query.session.delete(self) - Trade.query.session.flush() + Trade.commit() + + @staticmethod + def commit(): + Trade.query.session.commit() @staticmethod def get_trades_proxy(*, pair: str = None, is_open: bool = None, diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index 245f7cdab..af904f693 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -49,7 +49,7 @@ class PairLocks(): ) if PairLocks.use_db: PairLock.query.session.add(lock) - PairLock.query.session.flush() + PairLock.query.session.commit() else: PairLocks.locks.append(lock) @@ -99,7 +99,7 @@ class PairLocks(): for lock in locks: lock.active = False if PairLocks.use_db: - PairLock.query.session.flush() + PairLock.query.session.commit() @staticmethod def is_global_lock(now: Optional[datetime] = None) -> bool: diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index bb4283406..4119b7f71 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -47,7 +47,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0): data = load_data( datadir=config.get('datadir'), pairs=pairs, - timeframe=config.get('timeframe', '5m'), + timeframe=config['timeframe'], timerange=timerange, startup_candles=startup_candles, data_format=config.get('dataformat_ohlcv', 'json'), @@ -56,7 +56,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0): if startup_candles and data: min_date, max_date = get_timerange(data) logger.info(f"Loading data from {min_date} to {max_date}") - timerange.adjust_start_if_necessary(timeframe_to_seconds(config.get('timeframe', '5m')), + timerange.adjust_start_if_necessary(timeframe_to_seconds(config['timeframe']), startup_candles, min_date) no_trades = False @@ -96,20 +96,34 @@ def add_indicators(fig, row, indicators: Dict[str, Dict], data: pd.DataFrame) -> Dict key must correspond to dataframe column. :param data: candlestick DataFrame """ + plot_kinds = { + 'scatter': go.Scatter, + 'bar': go.Bar, + } for indicator, conf in indicators.items(): logger.debug(f"indicator {indicator} with config {conf}") if indicator in data: kwargs = {'x': data['date'], 'y': data[indicator].values, - 'mode': 'lines', 'name': indicator } - if 'color' in conf: - kwargs.update({'line': {'color': conf['color']}}) - scatter = go.Scatter( - **kwargs - ) - fig.add_trace(scatter, row, 1) + + plot_type = conf.get('type', 'scatter') + color = conf.get('color') + if plot_type == 'bar': + kwargs.update({'marker_color': color or 'DarkSlateGrey', + 'marker_line_color': color or 'DarkSlateGrey'}) + else: + if color: + kwargs.update({'line': {'color': color}}) + kwargs['mode'] = 'lines' + if plot_type != 'scatter': + logger.warning(f'Indicator {indicator} has unknown plot trace kind {plot_type}' + f', assuming "scatter".') + + kwargs.update(conf.get('plotly', {})) + trace = plot_kinds[plot_type](**kwargs) + fig.add_trace(trace, row, 1) else: logger.info( 'Indicator "%s" ignored. Reason: This indicator is not found ' @@ -274,8 +288,8 @@ def plot_area(fig, row: int, data: pd.DataFrame, indicator_a: str, :param fig: Plot figure to append to :param row: row number for this plot :param data: candlestick DataFrame - :param indicator_a: indicator name as populated in stragetie - :param indicator_b: indicator name as populated in stragetie + :param indicator_a: indicator name as populated in strategy + :param indicator_b: indicator name as populated in strategy :param label: label for the filled area :param fill_color: color to be used for the filled area :return: fig with added filled_traces plot @@ -569,6 +583,9 @@ def plot_profit(config: Dict[str, Any]) -> None: But should be somewhat proportional, and therefor useful in helping out to find a good algorithm. """ + if 'timeframe' not in config: + raise OperationalException('Timeframe must be set in either config or via --timeframe.') + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) plot_elements = init_plotscript(config, list(exchange.markets)) trades = plot_elements['trades'] @@ -585,7 +602,8 @@ def plot_profit(config: Dict[str, Any]) -> None: # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'], - trades, config.get('timeframe', '5m'), + trades, config['timeframe'], config.get('stake_currency', '')) store_plot_file(fig, filename='freqtrade-profit-plot.html', - directory=config['user_data_dir'] / 'plot', auto_open=True) + directory=config['user_data_dir'] / 'plot', + auto_open=config.get('plot_auto_open', False)) diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index bf474cb21..46a289ae6 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -19,7 +19,7 @@ class PerformanceFilter(IPairList): def needstickers(self) -> bool: """ Boolean property defining if tickers are necessary. - If no Pairlist requries tickers, an empty List is passed + If no Pairlist requires tickers, an empty List is passed as tickers argument to filter_pairlist """ return False diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index bc617a1db..5ae8e3e9f 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -20,9 +20,9 @@ logger = logging.getLogger(__name__) class VolatilityFilter(IPairList): - ''' + """ Filters pairs by volatility - ''' + """ def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index d1cdd2c5b..03f4760b8 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -83,7 +83,8 @@ class PairListManager(): pairlist = self._pairlist_handlers[0].gen_pairlist(tickers) # Process all Pairlist Handlers in the chain - for pairlist_handler in self._pairlist_handlers: + # except for the first one, which is the generator. + for pairlist_handler in self._pairlist_handlers[1:]: pairlist = pairlist_handler.filter_pairlist(pairlist, tickers) # Validation against blacklist happens after the chain of Pairlist Handlers diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index a8edd4e4b..f33e5b4bc 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -15,11 +15,11 @@ logger = logging.getLogger(__name__) class ProtectionManager(): - def __init__(self, config: dict) -> None: + def __init__(self, config: Dict, protections: List) -> None: self._config = config self._protection_handlers: List[IProtection] = [] - for protection_handler_config in self._config.get('protections', []): + for protection_handler_config in protections: protection_handler = ProtectionResolver.load_protection( protection_handler_config['method'], config=config, diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 635c0be04..45d393411 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -3,9 +3,9 @@ import logging from datetime import datetime, timedelta from typing import Any, Dict +from freqtrade.enums import SellType from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn -from freqtrade.strategy.interface import SellType logger = logging.getLogger(__name__) diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py index ed6715d15..4dfbf445b 100644 --- a/freqtrade/resolvers/exchange_resolver.py +++ b/freqtrade/resolvers/exchange_resolver.py @@ -21,6 +21,7 @@ class ExchangeResolver(IResolver): def load_exchange(exchange_name: str, config: dict, validate: bool = True) -> Exchange: """ Load the custom class from config parameter + :param exchange_name: name of the Exchange to load :param config: configuration dictionary """ # Map exchange name to avoid duplicate classes for identical exchanges diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index b51795e9e..2cccec70a 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -58,6 +58,9 @@ class IResolver: # Generate spec based on absolute path # Pass object_name as first argument to have logging print a reasonable name. spec = importlib.util.spec_from_file_location(object_name or "", str(module_path)) + if not spec: + return iter([None]) + module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints @@ -91,6 +94,9 @@ class IResolver: if not str(entry).endswith('.py'): logger.debug('Ignoring %s', entry) continue + if entry.is_symlink() and not entry.is_file(): + logger.debug('Ignoring broken symlink %s', entry) + continue module_path = entry.resolve() obj = next(cls._get_valid_object(module_path, object_name), None) @@ -129,7 +135,7 @@ class IResolver: extra_dir: Optional[str] = None) -> Any: """ Search and loads the specified object as configured in hte child class. - :param objectname: name of the module to import + :param object_name: name of the module to import :param config: configuration dictionary :param extra_dir: additional directory to search for the given pairlist :raises: OperationalException if the class is invalid or does not exist. @@ -157,7 +163,7 @@ class IResolver: :param directory: Path to search :param enum_failed: If True, will return None for modules which fail. Otherwise, failing modules are skipped. - :return: List of dicts containing 'name', 'class' and 'location' entires + :return: List of dicts containing 'name', 'class' and 'location' entries """ logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'") objects = [] diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 05fbac10d..e76d1e3e5 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -6,7 +6,6 @@ This module load custom strategies import logging import tempfile from base64 import urlsafe_b64decode -from collections import OrderedDict from inspect import getfullargspec from pathlib import Path from typing import Any, Dict, Optional @@ -114,7 +113,9 @@ class StrategyResolver(IResolver): - Strategy - default (if not None) """ - if attribute in config: + if (attribute in config + and not isinstance(getattr(type(strategy), 'my_property', None), property)): + # Ensure Properties are not overwritten setattr(strategy, attribute, config[attribute]) logger.info("Override strategy '%s' with value in config file: %s.", attribute, config[attribute]) @@ -139,7 +140,7 @@ class StrategyResolver(IResolver): # Sort and apply type conversions if hasattr(strategy, 'minimal_roi'): - strategy.minimal_roi = OrderedDict(sorted( + strategy.minimal_roi = dict(sorted( {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), key=lambda t: t[0])) if hasattr(strategy, 'stoploss'): diff --git a/freqtrade/rpc/__init__.py b/freqtrade/rpc/__init__.py index 0a0130ca7..957565e2c 100644 --- a/freqtrade/rpc/__init__.py +++ b/freqtrade/rpc/__init__.py @@ -1,3 +1,3 @@ # flake8: noqa: F401 -from .rpc import RPC, RPCException, RPCHandler, RPCMessageType +from .rpc import RPC, RPCException, RPCHandler from .rpc_manager import RPCManager diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index e907b92f0..965664028 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -162,8 +162,8 @@ def delete_lock_pair(payload: DeleteLockRequest, rpc: RPC = Depends(get_rpc)): @router.get('/logs', response_model=Logs, tags=['info']) -def logs(limit: Optional[int] = None, rpc: RPC = Depends(get_rpc)): - return rpc._rpc_get_logs(limit) +def logs(limit: Optional[int] = None): + return RPC._rpc_get_logs(limit) @router.post('/start', response_model=StatusMsg, tags=['botcontrol']) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 8a5c958e9..a43d4abe6 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -115,14 +115,12 @@ class ApiServer(RPCHandler): logger.info('Starting Local Rest Server.') verbosity = self._config['api_server'].get('verbosity', 'error') - log_config = uvicorn.config.LOGGING_CONFIG - # Change logging of access logs to stderr - log_config["handlers"]["access"]["stream"] = log_config["handlers"]["default"]["stream"] + uvconfig = uvicorn.Config(self.app, port=rest_port, host=rest_ip, use_colors=False, - log_config=log_config, + log_config=None, access_log=True if verbosity != 'error' else False, ) try: diff --git a/freqtrade/rpc/fiat_convert.py b/freqtrade/rpc/fiat_convert.py index 5ae20afa1..199e6a7db 100644 --- a/freqtrade/rpc/fiat_convert.py +++ b/freqtrade/rpc/fiat_convert.py @@ -102,7 +102,7 @@ class CryptoToFiatConverter: inverse = True symbol = f"{crypto_symbol}/{fiat_symbol}" - # Check if the fiat convertion you want is supported + # Check if the fiat conversion you want is supported if not self._is_supported_fiat(fiat=fiat_symbol): raise ValueError(f'The fiat {fiat_symbol} is not supported.') @@ -135,7 +135,7 @@ class CryptoToFiatConverter: :param fiat_symbol: FIAT currency you want to convert to (e.g usd) :return: float, price of the crypto-currency in Fiat """ - # Check if the fiat convertion you want is supported + # Check if the fiat conversion you want is supported if not self._is_supported_fiat(fiat=fiat_symbol): raise ValueError(f'The fiat {fiat_symbol} is not supported.') @@ -146,7 +146,7 @@ class CryptoToFiatConverter: if self._cryptomap == {}: if self._backoff <= datetime.datetime.now().timestamp(): self._load_cryptomap() - # return 0.0 if we still dont have data to check, no reason to proceed + # return 0.0 if we still don't have data to check, no reason to proceed if self._cryptomap == {}: return 0.0 else: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 3f26619a9..add66d576 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -4,7 +4,6 @@ This module contains class to define a RPC communications import logging from abc import abstractmethod from datetime import date, datetime, timedelta, timezone -from enum import Enum from math import isnan from typing import Any, Dict, List, Optional, Tuple, Union @@ -15,6 +14,7 @@ from pandas import DataFrame from freqtrade.configuration.timerange import TimeRange from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT from freqtrade.data.history import load_data +from freqtrade.enums import SellType, State from freqtrade.exceptions import ExchangeError, PricingError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.loggers import bufferHandler @@ -23,31 +23,12 @@ from freqtrade.persistence import PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter -from freqtrade.state import State -from freqtrade.strategy.interface import SellCheckTuple, SellType +from freqtrade.strategy.interface import SellCheckTuple logger = logging.getLogger(__name__) -class RPCMessageType(Enum): - STATUS = 'status' - WARNING = 'warning' - STARTUP = 'startup' - BUY = 'buy' - BUY_FILL = 'buy_fill' - BUY_CANCEL = 'buy_cancel' - SELL = 'sell' - SELL_FILL = 'sell_fill' - SELL_CANCEL = 'sell_cancel' - - def __repr__(self): - return self.value - - def __str__(self): - return self.value - - class RPCException(Exception): """ Should be raised with a rpc-formatted message in an _rpc_* method @@ -171,7 +152,7 @@ class RPC: # calculate profit and send message to user if trade.is_open: try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (ExchangeError, PricingError): current_rate = NAN else: @@ -199,9 +180,9 @@ class RPC: base_currency=self._freqtrade.config['stake_currency'], close_profit=trade.close_profit if trade.close_profit is not None else None, current_rate=current_rate, - current_profit=current_profit, # Deprectated - current_profit_pct=round(current_profit * 100, 2), # Deprectated - current_profit_abs=current_profit_abs, # Deprectated + current_profit=current_profit, # Deprecated + current_profit_pct=round(current_profit * 100, 2), # Deprecated + current_profit_abs=current_profit_abs, # Deprecated profit_ratio=current_profit, profit_pct=round(current_profit * 100, 2), profit_abs=current_profit_abs, @@ -230,7 +211,7 @@ class RPC: for trade in trades: # calculate profit and send message to user try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (PricingError, ExchangeError): current_rate = NAN trade_percent = (100 * trade.calc_profit_ratio(current_rate)) @@ -355,9 +336,12 @@ class RPC: return {'sell_reasons': sell_reasons, 'durations': durations} def _rpc_trade_statistics( - self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: + self, stake_currency: str, fiat_display_currency: str, + start_date: datetime = datetime.fromtimestamp(0)) -> Dict[str, Any]: """ Returns cumulative profit statistics """ - trades = Trade.get_trades().order_by(Trade.id).all() + trade_filter = ((Trade.is_open.is_(False) & (Trade.close_date >= start_date)) | + Trade.is_open.is_(True)) + trades = Trade.get_trades(trade_filter).order_by(Trade.id).all() profit_all_coin = [] profit_all_ratio = [] @@ -386,7 +370,7 @@ class RPC: else: # Get current rate try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (PricingError, ExchangeError): current_rate = NAN profit_ratio = trade.calc_profit_ratio(rate=current_rate) @@ -556,7 +540,7 @@ class RPC: if not fully_canceled: # Get current rate and execute sell - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL) self._freqtrade.execute_sell(trade, current_rate, sell_reason) # ---- EOF def _exec_forcesell ---- @@ -569,7 +553,7 @@ class RPC: # Execute sell for all open orders for trade in Trade.get_open_trades(): _exec_forcesell(trade) - Trade.query.session.flush() + Trade.commit() self._freqtrade.wallets.update() return {'result': 'Created sell orders for all open trades.'} @@ -582,7 +566,7 @@ class RPC: raise RPCException('invalid argument') _exec_forcesell(trade) - Trade.query.session.flush() + Trade.commit() self._freqtrade.wallets.update() return {'result': f'Created sell order for trade {trade_id}.'} @@ -615,6 +599,7 @@ class RPC: # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price, forcebuy=True): + Trade.commit() trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first() return trade else: @@ -705,8 +690,7 @@ class RPC: lock.active = False lock.lock_end_time = datetime.now(timezone.utc) - # session is always the same - PairLock.query.session.flush() + PairLock.query.session.commit() return self._rpc_locks() @@ -840,8 +824,11 @@ class RPC: ) if pair not in _data: raise RPCException(f"No data for {pair}, {timeframe} in {timerange} found.") + from freqtrade.data.dataprovider import DataProvider from freqtrade.resolvers.strategy_resolver import StrategyResolver strategy = StrategyResolver.load_strategy(config) + strategy.dp = DataProvider(config, exchange=None, pairlists=None) + df_analyzed = strategy.analyze_ticker(_data[pair], {'pair': pair}) return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe, diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index f819b55b4..18ed68041 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -4,7 +4,8 @@ This module contains class to manage RPC communications (Telegram, Slack, ...) import logging from typing import Any, Dict, List -from freqtrade.rpc import RPC, RPCHandler, RPCMessageType +from freqtrade.enums import RPCMessageType +from freqtrade.rpc import RPC, RPCHandler logger = logging.getLogger(__name__) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index b9e90dc8d..d857e46a9 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -5,25 +5,27 @@ This module manage Telegram communication """ import json import logging -from datetime import timedelta +import re +from datetime import date, datetime, timedelta from html import escape from itertools import chain from math import isnan -from typing import Any, Callable, Dict, List, Optional, Union, cast +from typing import Any, Callable, Dict, List, Optional, Union import arrow from tabulate import tabulate -from telegram import (InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, ParseMode, - ReplyKeyboardMarkup, Update) -from telegram.error import NetworkError, TelegramError +from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, + ParseMode, ReplyKeyboardMarkup, Update) +from telegram.error import BadRequest, NetworkError, TelegramError from telegram.ext import CallbackContext, CallbackQueryHandler, CommandHandler, Updater from telegram.utils.helpers import escape_markdown from freqtrade.__init__ import __version__ from freqtrade.constants import DUST_PER_COIN +from freqtrade.enums import RPCMessageType from freqtrade.exceptions import OperationalException from freqtrade.misc import chunks, round_coin_value -from freqtrade.rpc import RPC, RPCException, RPCHandler, RPCMessageType +from freqtrade.rpc import RPC, RPCException, RPCHandler logger = logging.getLogger(__name__) @@ -45,16 +47,20 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: update = kwargs.get('update') or args[0] # Reject unauthorized messages - chat_id = int(self._config['telegram']['chat_id']) + if update.callback_query: + cchat_id = int(update.callback_query.message.chat.id) + else: + cchat_id = int(update.message.chat_id) - if int(update.message.chat_id) != chat_id: + chat_id = int(self._config['telegram']['chat_id']) + if cchat_id != chat_id: logger.info( 'Rejected unauthorized message from: %s', update.message.chat_id ) return wrapper - logger.info( + logger.debug( 'Executing handler: %s for chat_id: %s', command_handler.__name__, chat_id @@ -89,7 +95,7 @@ class Telegram(RPCHandler): Validates the keyboard configuration from telegram config section. """ - self._keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = [ + self._keyboard: List[List[Union[str, KeyboardButton]]] = [ ['/daily', '/profit', '/balance'], ['/status', '/status table', '/performance'], ['/count', '/start', '/stop', '/help'] @@ -97,25 +103,29 @@ class Telegram(RPCHandler): # do not allow commands with mandatory arguments and critical cmds # like /forcesell and /forcebuy # TODO: DRY! - its not good to list all valid cmds here. But otherwise - # this needs refacoring of the whole telegram module (same + # this needs refactoring of the whole telegram module (same # problem in _help()). - valid_keys: List[str] = ['/start', '/stop', '/status', '/status table', - '/trades', '/profit', '/performance', '/daily', - '/stats', '/count', '/locks', '/balance', - '/stopbuy', '/reload_config', '/show_config', - '/logs', '/whitelist', '/blacklist', '/edge', - '/help', '/version'] + valid_keys: List[str] = [r'/start$', r'/stop$', r'/status$', r'/status table$', + r'/trades$', r'/performance$', r'/daily$', r'/daily \d+$', + r'/profit$', r'/profit \d+', + r'/stats$', r'/count$', r'/locks$', r'/balance$', + r'/stopbuy$', r'/reload_config$', r'/show_config$', + r'/logs$', r'/whitelist$', r'/blacklist$', r'/edge$', + r'/forcebuy$', r'/help$', r'/version$'] + # Create keys for generation + valid_keys_print = [k.replace('$', '') for k in valid_keys] # custom keyboard specified in config.json cust_keyboard = self._config['telegram'].get('keyboard', []) if cust_keyboard: + combined = "(" + ")|(".join(valid_keys) + ")" # check for valid shortcuts invalid_keys = [b for b in chain.from_iterable(cust_keyboard) - if b not in valid_keys] + if not re.match(combined, b)] if len(invalid_keys): err_msg = ('config.telegram.keyboard: Invalid commands for ' f'custom Telegram keyboard: {invalid_keys}' - f'\nvalid commands are: {valid_keys}') + f'\nvalid commands are: {valid_keys_print}') raise OperationalException(err_msg) else: self._keyboard = cust_keyboard @@ -158,8 +168,21 @@ class Telegram(RPCHandler): CommandHandler('help', self._help), CommandHandler('version', self._version), ] + callbacks = [ + CallbackQueryHandler(self._status_table, pattern='update_status_table'), + CallbackQueryHandler(self._daily, pattern='update_daily'), + CallbackQueryHandler(self._profit, pattern='update_profit'), + CallbackQueryHandler(self._balance, pattern='update_balance'), + CallbackQueryHandler(self._performance, pattern='update_performance'), + CallbackQueryHandler(self._count, pattern='update_count'), + CallbackQueryHandler(self._forcebuy_inline), + ] for handle in handles: self._updater.dispatcher.add_handler(handle) + + for callback in callbacks: + self._updater.dispatcher.add_handler(callback) + self._updater.start_polling( bootstrap_retries=-1, timeout=30, @@ -171,11 +194,6 @@ class Telegram(RPCHandler): [h.command for h in handles] ) - self._current_callback_query_handler: Optional[CallbackQueryHandler] = None - self._callback_query_handlers = { - 'forcebuy': CallbackQueryHandler(self._forcebuy_inline) - } - def cleanup(self) -> None: """ Stops all running telegram threads. @@ -211,66 +229,83 @@ class Telegram(RPCHandler): msg['emoji'] = self._get_sell_emoji(msg) - message = ("{emoji} *{exchange}:* Selling {pair} (#{trade_id})\n" - "*Amount:* `{amount:.8f}`\n" - "*Open Rate:* `{open_rate:.8f}`\n" - "*Current Rate:* `{current_rate:.8f}`\n" - "*Close Rate:* `{limit:.8f}`\n" - "*Sell Reason:* `{sell_reason}`\n" - "*Duration:* `{duration} ({duration_min:.1f} min)`\n" - "*Profit:* `{profit_percent:.2f}%`").format(**msg) - # Check if all sell properties are available. # This might not be the case if the message origin is triggered by /forcesell if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency']) and self._rpc._fiat_converter): msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount( msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) - message += (' `({gain}: {profit_amount:.8f} {stake_currency}' - ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg) + msg['profit_extra'] = (' ({gain}: {profit_amount:.8f} {stake_currency}' + ' / {profit_fiat:.3f} {fiat_currency})').format(**msg) + else: + msg['profit_extra'] = '' + + message = ("{emoji} *{exchange}:* Selling {pair} (#{trade_id})\n" + "*Profit:* `{profit_percent:.2f}%{profit_extra}`\n" + "*Sell Reason:* `{sell_reason}`\n" + "*Duration:* `{duration} ({duration_min:.1f} min)`\n" + "*Amount:* `{amount:.8f}`\n" + "*Open Rate:* `{open_rate:.8f}`\n" + "*Current Rate:* `{current_rate:.8f}`\n" + "*Close Rate:* `{limit:.8f}`").format(**msg) + return message def send_msg(self, msg: Dict[str, Any]) -> None: """ Send a message to telegram channel """ - noti = self._config['telegram'].get('notification_settings', {} - ).get(str(msg['type']), 'on') + default_noti = 'on' + + msg_type = msg['type'] + noti = '' + if msg_type == RPCMessageType.SELL: + sell_noti = self._config['telegram'] \ + .get('notification_settings', {}).get(str(msg_type), {}) + # For backward compatibility sell still can be string + if isinstance(sell_noti, str): + noti = sell_noti + else: + noti = sell_noti.get(str(msg['sell_reason']), default_noti) + else: + noti = self._config['telegram'] \ + .get('notification_settings', {}).get(str(msg_type), default_noti) + if noti == 'off': - logger.info(f"Notification '{msg['type']}' not sent.") + logger.info(f"Notification '{msg_type}' not sent.") # Notification disabled return - if msg['type'] == RPCMessageType.BUY: + if msg_type == RPCMessageType.BUY: message = self._format_buy_msg(msg) - elif msg['type'] in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL): - msg['message_side'] = 'buy' if msg['type'] == RPCMessageType.BUY_CANCEL else 'sell' + elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL): + msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell' message = ("\N{WARNING SIGN} *{exchange}:* " "Cancelling open {message_side} Order for {pair} (#{trade_id}). " "Reason: {reason}.".format(**msg)) - elif msg['type'] == RPCMessageType.BUY_FILL: + elif msg_type == RPCMessageType.BUY_FILL: message = ("\N{LARGE CIRCLE} *{exchange}:* " "Buy order for {pair} (#{trade_id}) filled " "for {open_rate}.".format(**msg)) - elif msg['type'] == RPCMessageType.SELL_FILL: + elif msg_type == RPCMessageType.SELL_FILL: message = ("\N{LARGE CIRCLE} *{exchange}:* " "Sell order for {pair} (#{trade_id}) filled " "for {close_rate}.".format(**msg)) - elif msg['type'] == RPCMessageType.SELL: + elif msg_type == RPCMessageType.SELL: message = self._format_sell_msg(msg) - elif msg['type'] == RPCMessageType.STATUS: + elif msg_type == RPCMessageType.STATUS: message = '*Status:* `{status}`'.format(**msg) - elif msg['type'] == RPCMessageType.WARNING: + elif msg_type == RPCMessageType.WARNING: message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) - elif msg['type'] == RPCMessageType.STARTUP: + elif msg_type == RPCMessageType.STARTUP: message = '{status}'.format(**msg) else: - raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) + raise NotImplementedError('Unknown message type: {}'.format(msg_type)) self._send_msg(message, disable_notification=(noti == 'silent')) @@ -386,7 +421,9 @@ class Telegram(RPCHandler): # insert separators line between Total lines = message.split("\n") message = "\n".join(lines[:-1] + [lines[1]] + [lines[-1]]) - self._send_msg(f"
{message}
", parse_mode=ParseMode.HTML) + self._send_msg(f"
{message}
", parse_mode=ParseMode.HTML, + reload_able=True, callback_path="update_status_table", + query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -424,7 +461,8 @@ class Telegram(RPCHandler): ], tablefmt='simple') message = f'Daily Profit over the last {timescale} days:\n
{stats_tab}
' - self._send_msg(message, parse_mode=ParseMode.HTML) + self._send_msg(message, parse_mode=ParseMode.HTML, reload_able=True, + callback_path="update_daily", query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -440,9 +478,20 @@ class Telegram(RPCHandler): stake_cur = self._config['stake_currency'] fiat_disp_cur = self._config.get('fiat_display_currency', '') + start_date = datetime.fromtimestamp(0) + timescale = None + try: + if context.args: + timescale = int(context.args[0]) - 1 + today_start = datetime.combine(date.today(), datetime.min.time()) + start_date = today_start - timedelta(days=timescale) + except (TypeError, ValueError, IndexError): + pass + stats = self._rpc._rpc_trade_statistics( stake_cur, - fiat_disp_cur) + fiat_disp_cur, + start_date) profit_closed_coin = stats['profit_closed_coin'] profit_closed_percent_mean = stats['profit_closed_percent_mean'] profit_closed_percent_sum = stats['profit_closed_percent_sum'] @@ -470,20 +519,23 @@ class Telegram(RPCHandler): else: markdown_msg = "`No closed trade` \n" - markdown_msg += (f"*ROI:* All trades\n" - f"∙ `{round_coin_value(profit_all_coin, stake_cur)} " - f"({profit_all_percent_mean:.2f}%) " - f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" - f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n" - f"*Total Trade Count:* `{trade_count}`\n" - f"*First Trade opened:* `{first_trade_date}`\n" - f"*Latest Trade opened:* `{latest_trade_date}\n`" - f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" - ) + markdown_msg += ( + f"*ROI:* All trades\n" + f"∙ `{round_coin_value(profit_all_coin, stake_cur)} " + f"({profit_all_percent_mean:.2f}%) " + f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" + f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n" + f"*Total Trade Count:* `{trade_count}`\n" + f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* " + f"`{first_trade_date}`\n" + f"*Latest Trade opened:* `{latest_trade_date}\n`" + f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" + ) if stats['closed_trade_count'] > 0: markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") - self._send_msg(markdown_msg) + self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit", + query=update.callback_query) @authorized_only def _stats(self, update: Update, context: CallbackContext) -> None: @@ -559,7 +611,7 @@ class Telegram(RPCHandler): curr_output = (f"*{curr['currency']}:* not showing <{balance_dust_level} " f"{curr['stake']} amount \n") - # Handle overflowing messsage length + # Handle overflowing message length if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH: self._send_msg(output) output = curr_output @@ -570,7 +622,8 @@ class Telegram(RPCHandler): f"\t`{result['stake']}: {result['total']: .8f}`\n" f"\t`{result['symbol']}: " f"{round_coin_value(result['value'], result['symbol'], False)}`\n") - self._send_msg(output) + self._send_msg(output, reload_able=True, callback_path="update_balance", + query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -677,10 +730,10 @@ class Telegram(RPCHandler): self._forcebuy_action(pair, price) else: whitelist = self._rpc._rpc_whitelist()['whitelist'] - pairs = [InlineKeyboardButton(pair, callback_data=pair) for pair in whitelist] - self._send_inline_msg("Which pair?", - keyboard=self._layout_inline_keyboard(pairs), - callback_query_handler='forcebuy') + pairs = [InlineKeyboardButton(text=pair, callback_data=pair) for pair in whitelist] + + self._send_msg(msg="Which pair?", + keyboard=self._layout_inline_keyboard(pairs)) @authorized_only def _trades(self, update: Update, context: CallbackContext) -> None: @@ -764,7 +817,9 @@ class Telegram(RPCHandler): else: output += stat_line - self._send_msg(output, parse_mode=ParseMode.HTML) + self._send_msg(output, parse_mode=ParseMode.HTML, + reload_able=True, callback_path="update_performance", + query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -784,7 +839,9 @@ class Telegram(RPCHandler): tablefmt='simple') message = "
{}
".format(message) logger.debug(message) - self._send_msg(message, parse_mode=ParseMode.HTML) + self._send_msg(message, parse_mode=ParseMode.HTML, + reload_able=True, callback_path="update_count", + query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -942,7 +999,8 @@ class Telegram(RPCHandler): " `pending buy orders are marked with an asterisk (*)`\n" " `pending sell orders are marked with a double asterisk (**)`\n" "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n" - "*/profit:* `Lists cumulative profit from all finished trades`\n" + "*/profit []:* `Lists cumulative profit from all finished trades, " + "over the last n days`\n" "*/forcesell |all:* `Instantly sells the given trade or all trades, " "regardless of profit`\n" f"{forcebuy_text if self._config.get('forcebuy_enable', False) else ''}" @@ -1015,29 +1073,42 @@ class Telegram(RPCHandler): f"*Current state:* `{val['state']}`" ) - def _send_inline_msg(self, msg: str, callback_query_handler, - parse_mode: str = ParseMode.MARKDOWN, disable_notification: bool = False, - keyboard: List[List[InlineKeyboardButton]] = None, ) -> None: - """ - Send given markdown message - :param msg: message - :param bot: alternative bot - :param parse_mode: telegram parse mode - :return: None - """ - if self._current_callback_query_handler: - self._updater.dispatcher.remove_handler(self._current_callback_query_handler) - self._current_callback_query_handler = self._callback_query_handlers[callback_query_handler] - self._updater.dispatcher.add_handler(self._current_callback_query_handler) + def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "", + reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None: + if reload_able: + reply_markup = InlineKeyboardMarkup([ + [InlineKeyboardButton("Refresh", callback_data=callback_path)], + ]) + else: + reply_markup = InlineKeyboardMarkup([[]]) + msg += "\nUpdated: {}".format(datetime.now().ctime()) + if not query.message: + return + chat_id = query.message.chat_id + message_id = query.message.message_id - self._send_msg(msg, parse_mode, disable_notification, - cast(List[List[Union[str, KeyboardButton, InlineKeyboardButton]]], keyboard), - reply_markup=InlineKeyboardMarkup) + try: + self._updater.bot.edit_message_text( + chat_id=chat_id, + message_id=message_id, + text=msg, + parse_mode=parse_mode, + reply_markup=reply_markup + ) + except BadRequest as e: + if 'not modified' in e.message.lower(): + pass + else: + logger.warning('TelegramError: %s', e.message) + except TelegramError as telegram_err: + logger.warning('TelegramError: %s! Giving up on that message.', telegram_err.message) def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN, disable_notification: bool = False, - keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = None, - reply_markup=ReplyKeyboardMarkup) -> None: + keyboard: List[List[InlineKeyboardButton]] = None, + callback_path: str = "", + reload_able: bool = False, + query: Optional[CallbackQuery] = None) -> None: """ Send given markdown message :param msg: message @@ -1045,9 +1116,19 @@ class Telegram(RPCHandler): :param parse_mode: telegram parse mode :return: None """ - if keyboard is None: - keyboard = self._keyboard - reply_markup = reply_markup(keyboard, resize_keyboard=True) + reply_markup: Union[InlineKeyboardMarkup, ReplyKeyboardMarkup] + if query: + self._update_msg(query=query, msg=msg, parse_mode=parse_mode, + callback_path=callback_path, reload_able=reload_able) + return + if reload_able and self._config['telegram'].get('reload', True): + reply_markup = InlineKeyboardMarkup([ + [InlineKeyboardButton("Refresh", callback_data=callback_path)]]) + else: + if keyboard is not None: + reply_markup = InlineKeyboardMarkup(keyboard, resize_keyboard=True) + else: + reply_markup = ReplyKeyboardMarkup(self._keyboard, resize_keyboard=True) try: try: self._updater.bot.send_message( diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 24e1348f1..b4c55649e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -6,7 +6,8 @@ from typing import Any, Dict from requests import RequestException, post -from freqtrade.rpc import RPC, RPCHandler, RPCMessageType +from freqtrade.enums import RPCMessageType +from freqtrade.rpc import RPC, RPCHandler logger = logging.getLogger(__name__) @@ -76,14 +77,13 @@ class Webhook(RPCHandler): def _send_msg(self, payload: dict) -> None: """do the actual call to the webhook""" - if self._format == 'form': - kwargs = {'data': payload} - elif self._format == 'json': - kwargs = {'json': payload} - else: - raise NotImplementedError('Unknown format: {}'.format(self._format)) - try: - post(self._url, **kwargs) + if self._format == 'form': + post(self._url, data=payload) + elif self._format == 'json': + post(self._url, json=payload) + else: + raise NotImplementedError('Unknown format: {}'.format(self._format)) + except RequestException as exc: logger.warning("Could not call webhook url. Exception: %s", exc) diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 7dee47d87..21a806202 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -14,8 +14,8 @@ with suppress(ImportError): from skopt.space import Integer, Real, Categorical from freqtrade.optimize.space import SKDecimal +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -242,7 +242,7 @@ class CategoricalParameter(BaseParameter): class HyperStrategyMixin(object): """ - A helper base class which allows HyperOptAuto class to reuse implementations of of buy/sell + A helper base class which allows HyperOptAuto class to reuse implementations of buy/sell strategy logic. """ @@ -258,7 +258,7 @@ class HyperStrategyMixin(object): def enumerate_parameters(self, category: str = None) -> Iterator[Tuple[str, BaseParameter]]: """ - Find all optimizeable parameters and return (name, attr) iterator. + Find all optimizable parameters and return (name, attr) iterator. :param category: :return: """ @@ -273,11 +273,12 @@ class HyperStrategyMixin(object): for par in params: yield par.name, par - def _detect_parameters(self, category: str) -> Iterator[Tuple[str, BaseParameter]]: + @classmethod + def detect_parameters(cls, category: str) -> Iterator[Tuple[str, BaseParameter]]: """ Detect all parameters for 'category' """ - for attr_name in dir(self): + for attr_name in dir(cls): if not attr_name.startswith('__'): # Ignore internals, not strictly necessary. - attr = getattr(self, attr_name) + attr = getattr(cls, attr_name) if issubclass(attr.__class__, BaseParameter): if (attr_name.startswith(category + '_') and attr.category is not None and attr.category != category): @@ -287,6 +288,19 @@ class HyperStrategyMixin(object): (attr_name.startswith(category + '_') and attr.category is None)): yield attr_name, attr + @classmethod + def detect_all_parameters(cls) -> Dict: + """ Detect all parameters and return them as a list""" + params: Dict = { + 'buy': list(cls.detect_parameters('buy')), + 'sell': list(cls.detect_parameters('sell')), + } + params.update({ + 'count': len(params['buy'] + params['sell']) + }) + + return params + def _load_hyper_params(self, hyperopt: bool = False) -> None: """ Load Hyperoptable parameters @@ -296,14 +310,14 @@ class HyperStrategyMixin(object): def _load_params(self, params: dict, space: str, hyperopt: bool = False) -> None: """ - Set optimizeable parameter values. + Set optimizable parameter values. :param params: Dictionary with new parameter values. """ if not params: logger.info(f"No params for {space} found, using default values.") param_container: List[BaseParameter] = getattr(self, f"ft_{space}_params") - for attr_name, attr in self._detect_parameters(space): + for attr_name, attr in self.detect_parameters(space): attr.name = attr_name attr.in_space = hyperopt and HyperoptTools.has_space(self.config, space) if not attr.category: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e2cde52eb..ea7d5079d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,6 @@ import logging import warnings from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone -from enum import Enum from typing import Dict, List, Optional, Tuple, Union import arrow @@ -14,6 +13,7 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums import SellType, SignalType from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date @@ -27,33 +27,6 @@ logger = logging.getLogger(__name__) CUSTOM_SELL_MAX_LENGTH = 64 -class SignalType(Enum): - """ - Enum to distinguish between buy and sell signals - """ - BUY = "buy" - SELL = "sell" - - -class SellType(Enum): - """ - Enum to distinguish between sell reasons - """ - ROI = "roi" - STOP_LOSS = "stop_loss" - STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange" - TRAILING_STOP_LOSS = "trailing_stop_loss" - SELL_SIGNAL = "sell_signal" - FORCE_SELL = "force_sell" - EMERGENCY_SELL = "emergency_sell" - CUSTOM_SELL = "custom_sell" - NONE = "" - - def __str__(self): - # explicitly convert to String to help with exporting data. - return self.value - - class SellCheckTuple(object): """ NamedTuple for Sell type + reason @@ -134,7 +107,7 @@ class IStrategy(ABC, HyperStrategyMixin): startup_candle_count: int = 0 # Protections - protections: List + protections: List = [] # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) @@ -297,7 +270,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param current_rate: Rate, calculated based on pricing settings in ask_strategy. :param current_profit: Current profit (as ratio), calculated based on current_rate. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: New stoploss value, relative to the currentrate + :return float: New stoploss value, relative to the current_rate """ return self.stoploss @@ -328,7 +301,7 @@ class IStrategy(ABC, HyperStrategyMixin): def informative_pairs(self) -> ListPairsWithTimeframes: """ Define additional, informative pair/interval combinations to be cached from the exchange. - These pair/interval combinations are non-tradeable, unless they are part + These pair/interval combinations are non-tradable, unless they are part of the whitelist as well. For more information, please consult the documentation :return: List of tuples in the format (pair, interval) @@ -376,7 +349,7 @@ class IStrategy(ABC, HyperStrategyMixin): The 2nd, optional parameter ensures that locks are applied until the new candle arrives, and not stop at 14:00:00 - while the next candle arrives at 14:00:02 leaving a gap of 2 seconds for a buy to happen on an old signal. - :param: pair: "Pair to check" + :param pair: "Pair to check" :param candle_date: Date of the last candle. Optional, defaults to current date :returns: locking state of the pair in question. """ @@ -480,18 +453,25 @@ class IStrategy(ABC, HyperStrategyMixin): """ Ensure dataframe (length, last candle) was not modified, and has all elements we need. """ + message_template = "Dataframe returned from strategy has mismatching {}." message = "" - if df_len != len(dataframe): - message = "length" + if dataframe is None: + message = "No dataframe returned (return statement missing?)." + elif 'buy' not in dataframe: + message = "Buy column not set." + elif 'sell' not in dataframe: + message = "Sell column not set." + elif df_len != len(dataframe): + message = message_template.format("length") elif df_close != dataframe["close"].iloc[-1]: - message = "last close price" + message = message_template.format("last close price") elif df_date != dataframe["date"].iloc[-1]: - message = "last date" + message = message_template.format("last date") if message: if self.disable_dataframe_checks: - logger.warning(f"Dataframe returned from strategy has mismatching {message}.") + logger.warning(message) else: - raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.") + raise StrategyError(message) def get_signal(self, pair: str, timeframe: str, dataframe: DataFrame) -> Tuple[bool, bool]: """ @@ -551,15 +531,14 @@ class IStrategy(ABC, HyperStrategyMixin): :param force_stoploss: Externally provided stoploss :return: True if trade should be sold, False otherwise """ - # Set current rate to low for backtesting sell - current_rate = low or rate + current_rate = rate current_profit = trade.calc_profit_ratio(current_rate) trade.adjust_min_max_rates(high or current_rate) stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, current_profit=current_profit, - force_stoploss=force_stoploss, high=high) + force_stoploss=force_stoploss, low=low, high=high) # Set current rate to high for backtesting sell current_rate = high or rate @@ -626,18 +605,21 @@ class IStrategy(ABC, HyperStrategyMixin): def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, current_profit: float, - force_stoploss: float, high: float = None) -> SellCheckTuple: + force_stoploss: float, low: float = None, + high: float = None) -> SellCheckTuple: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not :param current_profit: current profit as ratio + :param low: Low value of this candle, only set in backtesting + :param high: High value of this candle, only set in backtesting """ stop_loss_value = force_stoploss if force_stoploss else self.stoploss # Initiate stoploss with open_rate. Does nothing if stoploss is already set. trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True) - if self.use_custom_stoploss: + if self.use_custom_stoploss and trade.stop_loss < (low or current_rate): stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None )(pair=trade.pair, trade=trade, current_time=current_time, @@ -650,7 +632,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: logger.warning("CustomStoploss function did not return valid stoploss") - if self.trailing_stop: + if self.trailing_stop and trade.stop_loss < (low or current_rate): # trailing stoploss handling sl_offset = self.trailing_stop_positive_offset @@ -670,7 +652,7 @@ class IStrategy(ABC, HyperStrategyMixin): # evaluate if the stoploss was hit if stoploss is not on exchange # in Dry-Run, this handles stoploss logic as well, as the logic will not be different to # regular stoploss handling. - if ((trade.stop_loss >= current_rate) and + if ((trade.stop_loss >= (low or current_rate)) and (not self.order_types.get('stoploss_on_exchange') or self.config['dry_run'])): sell_type = SellType.STOP_LOSS @@ -679,7 +661,7 @@ class IStrategy(ABC, HyperStrategyMixin): if trade.initial_stop_loss != trade.stop_loss: sell_type = SellType.TRAILING_STOP_LOSS logger.debug( - f"{trade.pair} - HIT STOP: current price at {current_rate:.6f}, " + f"{trade.pair} - HIT STOP: current price at {(low or current_rate):.6f}, " f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at {trade.initial_stop_loss:.6f}, " f"trade opened at {trade.open_rate:.6f}") @@ -751,7 +733,8 @@ class IStrategy(ABC, HyperStrategyMixin): Based on TA indicators, populates the buy signal for the given dataframe This method should not be overridden. :param dataframe: DataFrame - :param pair: Additional information, like the currently traded pair + :param metadata: Additional information dictionary, with details like the + currently traded pair :return: DataFrame with buy column """ logger.debug(f"Populating buy signals for pair {metadata.get('pair')}.") @@ -768,7 +751,8 @@ class IStrategy(ABC, HyperStrategyMixin): Based on TA indicators, populates the sell signal for the given dataframe This method should not be overridden. :param dataframe: DataFrame - :param pair: Additional information, like the currently traded pair + :param metadata: Additional information dictionary, with details like the + currently traded pair :return: DataFrame with sell column """ logger.debug(f"Populating sell signals for pair {metadata.get('pair')}.") diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index e51feff1e..282b2f8e2 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -329,7 +329,7 @@ class SampleStrategy(IStrategy): """ # first check if dataprovider is available if self.dp: - if self.dp.runmode in ('live', 'dry_run'): + if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 57d2ca665..a497b47cb 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -199,7 +199,7 @@ dataframe['htleadsine'] = hilbert['leadsine'] """ # first check if dataprovider is available if self.dp: - if self.dp.runmode in ('live', 'dry_run'): + if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] diff --git a/freqtrade/templates/subtemplates/indicators_minimal.j2 b/freqtrade/templates/subtemplates/indicators_minimal.j2 index 7d75b4610..90f4f4d4a 100644 --- a/freqtrade/templates/subtemplates/indicators_minimal.j2 +++ b/freqtrade/templates/subtemplates/indicators_minimal.j2 @@ -10,7 +10,7 @@ dataframe['rsi'] = ta.RSI(dataframe) """ # first check if dataprovider is available if self.dp: - if self.dp.runmode in ('live', 'dry_run'): + if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index d5f979c24..1b2ec4550 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -8,10 +8,10 @@ from typing import Any, Dict, NamedTuple import arrow from freqtrade.constants import UNLIMITED_STAKE_AMOUNT +from freqtrade.enums import RunMode from freqtrade.exceptions import DependencyException from freqtrade.exchange import Exchange from freqtrade.persistence import LocalTrade, Trade -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index ec9331eef..5c0de86ff 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -11,9 +11,9 @@ import sdnotify from freqtrade import __version__, constants from freqtrade.configuration import Configuration +from freqtrade.enums import State from freqtrade.exceptions import OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.state import State logger = logging.getLogger(__name__) @@ -61,7 +61,7 @@ class Worker: def _notify(self, message: str) -> None: """ - Removes the need to verify in all occurances if sd_notify is enabled + Removes the need to verify in all occurrences if sd_notify is enabled :param message: Message to send to systemd if it's enabled. """ if self._sd_notify: diff --git a/mkdocs.yml b/mkdocs.yml index 2520ca929..854939ca0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,7 @@ site_name: Freqtrade +site_url: https://www.freqtrade.io/ repo_url: https://github.com/freqtrade/freqtrade +use_directory_urls: True nav: - Home: index.md - Quickstart with Docker: docker_quickstart.md @@ -40,10 +42,20 @@ theme: name: material logo: 'images/logo.png' favicon: 'images/logo.png' - custom_dir: 'docs' + custom_dir: 'docs/overrides' palette: - primary: 'blue grey' - accent: 'tear' + - scheme: default + primary: 'blue grey' + accent: 'tear' + toggle: + icon: material/toggle-switch-off-outline + name: Switch to dark mode + - scheme: slate + primary: 'blue grey' + accent: 'tear' + toggle: + icon: material/toggle-switch-off-outline + name: Switch to dark mode extra_css: - 'stylesheets/ft.extra.css' extra_javascript: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..f0637d8c6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[tool.black] +line-length = 100 +exclude = ''' +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + )/ + # Exclude vendor directory + | vendor +) +''' + +[tool.isort] +line_length = 100 +multi_line_output=0 +lines_after_imports=2 + +[build-system] +requires = ["setuptools >= 46.4.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/requirements-dev.txt b/requirements-dev.txt index 4fbf21260..30044058b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,17 +3,23 @@ -r requirements-plot.txt -r requirements-hyperopt.txt -coveralls==3.0.1 +coveralls==3.1.0 flake8==3.9.2 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.3.0 -mypy==0.812 +mypy==0.902 pytest==6.2.4 pytest-asyncio==0.15.1 -pytest-cov==2.12.0 +pytest-cov==2.12.1 pytest-mock==3.6.1 pytest-random-order==1.0.4 isort==5.8.0 # Convert jupyter notebooks to markdown documents nbconvert==6.0.7 + +# mypy types +types-cachetools==0.1.8 +types-filelock==0.1.4 +types-requests==0.1.13 +types-tabulate==0.1.1 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 5e7e9d9d2..83e23e3ec 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.6.3 +scipy==1.7.0 scikit-learn==0.24.2 scikit-optimize==0.8.1 filelock==3.0.12 diff --git a/requirements.txt b/requirements.txt index 512b2a588..039441f36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,25 +1,25 @@ numpy==1.20.3 pandas==1.2.4 -ccxt==1.50.48 +ccxt==1.51.77 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.15 -python-telegram-bot==13.5 +SQLAlchemy==1.4.18 +python-telegram-bot==13.6 arrow==1.1.0 cachetools==4.2.2 requests==2.25.1 -urllib3==1.26.4 +urllib3==1.26.5 wrapt==1.12.1 jsonschema==3.2.0 TA-Lib==0.4.20 technical==1.3.0 tabulate==0.8.9 -pycoingecko==2.0.0 +pycoingecko==2.2.0 jinja2==3.0.1 tables==3.6.1 -blosc==1.10.2 +blosc==1.10.4 # find first, C search in arrays py_find_1st==1.1.5 @@ -31,8 +31,8 @@ python-rapidjson==1.0 sdnotify==0.3.2 # API Server -fastapi==0.65.1 -uvicorn==0.13.4 +fastapi==0.65.2 +uvicorn==0.14.0 pyjwt==2.1.0 aiofiles==0.7.0 @@ -40,4 +40,4 @@ aiofiles==0.7.0 colorama==0.4.4 # Building config files interactively questionary==1.9.0 -prompt-toolkit==3.0.18 +prompt-toolkit==3.0.19 diff --git a/setup.cfg b/setup.cfg index be2cd450c..b311c94da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,43 @@ +[metadata] +name = freqtrade +version = attr: freqtrade.__version__ +author = Freqtrade Team +author_email = freqtrade@protonmail.com +description = Freqtrade - Crypto Trading Bot +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/freqtrade/freqtrade +project_urls = + Bug Tracker = https://github.com/freqtrade/freqtrade/issues +license = GPLv3 +classifiers = + Environment :: Console + Intended Audience :: Science/Research + License :: OSI Approved :: GNU General Public License v3 (GPLv3) + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Operating System :: MacOS + Operating System :: Unix + Topic :: Office/Business :: Financial :: Investment + + +[options] +zip_safe = False +include_package_data = True +tests_require = + pytest + pytest-asyncio + pytest-cov + pytest-mock + +packages = find: +python_requires = >=3.6 + +[options.entry_points] +console_scripts = + freqtrade = freqtrade.main:main + [flake8] #ignore = max-line-length = 100 @@ -8,11 +48,6 @@ exclude = .eggs, user_data, -[isort] -line_length=100 -multi_line_output=0 -lines_after_imports=2 - [mypy] ignore_missing_imports = True diff --git a/setup.py b/setup.py index 54a2e01b5..727c40c7c 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,7 @@ -from sys import version_info - from setuptools import setup -if version_info.major == 3 and version_info.minor < 7 or \ - version_info.major < 3: - print('Your Python interpreter must be 3.7 or greater!') - exit(1) - -from pathlib import Path # noqa: E402 - -from freqtrade import __version__ # noqa: E402 - - -readme_file = Path(__file__).parent / "README.md" -readme_long = "Crypto Trading Bot" -if readme_file.is_file(): - readme_long = (Path(__file__).parent / "README.md").read_text() - # Requirements used for submodules -api = ['fastapi', 'uvicorn', 'pyjwt', 'aiofiles'] plot = ['plotly>=4.0'] hyperopt = [ 'scipy', @@ -51,69 +33,51 @@ jupyter = [ 'nbconvert', ] -all_extra = api + plot + develop + jupyter + hyperopt +all_extra = plot + develop + jupyter + hyperopt -setup(name='freqtrade', - version=__version__, - description='Crypto Trading Bot', - long_description=readme_long, - long_description_content_type="text/markdown", - url='https://github.com/freqtrade/freqtrade', - author='Freqtrade Team', - author_email='michael.egger@tsn.at', - license='GPLv3', - packages=['freqtrade'], - setup_requires=['pytest-runner', 'numpy'], - tests_require=['pytest', 'pytest-asyncio', 'pytest-cov', 'pytest-mock', ], - install_requires=[ - # from requirements.txt - 'ccxt>=1.24.96', - 'SQLAlchemy', - 'python-telegram-bot>=13.4', - 'arrow>=0.17.0', - 'cachetools', - 'requests', - 'urllib3', - 'wrapt', - 'jsonschema', - 'TA-Lib', - 'technical', - 'tabulate', - 'pycoingecko', - 'py_find_1st', - 'python-rapidjson', - 'sdnotify', - 'colorama', - 'jinja2', - 'questionary', - 'prompt-toolkit', - 'numpy', - 'pandas', - 'tables', - 'blosc', - ], - extras_require={ - 'api': api, - 'dev': all_extra, - 'plot': plot, - 'jupyter': jupyter, - 'hyperopt': hyperopt, - 'all': all_extra, - }, - include_package_data=True, - zip_safe=False, - entry_points={ - 'console_scripts': [ - 'freqtrade = freqtrade.main:main', - ], - }, - classifiers=[ - 'Environment :: Console', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Operating System :: MacOS', - 'Operating System :: Unix', - 'Topic :: Office/Business :: Financial :: Investment', - ]) +setup( + tests_require=[ + 'pytest', + 'pytest-asyncio', + 'pytest-cov', + 'pytest-mock', + ], + install_requires=[ + # from requirements.txt + 'ccxt>=1.50.48', + 'SQLAlchemy', + 'python-telegram-bot>=13.4', + 'arrow>=0.17.0', + 'cachetools', + 'requests', + 'urllib3', + 'wrapt', + 'jsonschema', + 'TA-Lib', + 'technical', + 'tabulate', + 'pycoingecko', + 'py_find_1st', + 'python-rapidjson', + 'sdnotify', + 'colorama', + 'jinja2', + 'questionary', + 'prompt-toolkit', + 'numpy', + 'pandas', + 'tables', + 'blosc', + 'fastapi', + 'uvicorn', + 'pyjwt', + 'aiofiles' + ], + extras_require={ + 'dev': all_extra, + 'plot': plot, + 'jupyter': jupyter, + 'hyperopt': hyperopt, + 'all': all_extra, + }, +) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 4d3937d87..47f298ad7 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1,3 +1,4 @@ +import json import re from io import BytesIO from pathlib import Path @@ -16,8 +17,8 @@ from freqtrade.commands import (start_convert_data, start_create_userdir, start_ from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) from tests.conftest_trades import MOCK_TRADE_COUNT @@ -914,16 +915,24 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): ] start_test_pairlist(get_args(args)) captured = capsys.readouterr() - assert re.match(r'Pairs for BTC: \n\["ETH/BTC","TKN/BTC","BLK/BTC","LTC/BTC","XRP/BTC"\]\n', - captured.out) + try: + json_pairs = json.loads(captured.out) + assert 'ETH/BTC' in json_pairs + assert 'TKN/BTC' in json_pairs + assert 'BLK/BTC' in json_pairs + assert 'LTC/BTC' in json_pairs + assert 'XRP/BTC' in json_pairs + except json.decoder.JSONDecodeError: + pytest.fail(f'Expected well formed JSON, but failed to parse: {captured.out}') def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, - saved_hyperopt_results_legacy): - for _ in (saved_hyperopt_results, saved_hyperopt_results_legacy): + saved_hyperopt_results_legacy, tmpdir): + csv_file = Path(tmpdir) / "test.csv" + for res in (saved_hyperopt_results, saved_hyperopt_results_legacy): mocker.patch( 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', - MagicMock(return_value=saved_hyperopt_results_legacy) + MagicMock(return_value=res) ) args = [ @@ -1139,17 +1148,19 @@ def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, "hyperopt-list", "--no-details", "--no-color", - "--export-csv", "test_file.csv", + "--export-csv", + str(csv_file), ] pargs = get_args(args) pargs['config'] = None start_hyperopt_list(pargs) captured = capsys.readouterr() log_has("CSV file created: test_file.csv", caplog) - f = Path("test_file.csv") - assert 'Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' in f.read_text() - assert f.is_file() - f.unlink() + assert csv_file.is_file() + line = csv_file.read_text() + assert ('Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' in line + or "Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,2 days 17:30:00,0.43662" in line) + csv_file.unlink() def test_hyperopt_show(mocker, capsys, saved_hyperopt_results): diff --git a/tests/conftest.py b/tests/conftest.py index ef2bd0613..6e00db274 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,11 +17,11 @@ from freqtrade import constants from freqtrade.commands import Arguments from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo +from freqtrade.enums import RunMode from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import LocalTrade, Trade, init_db from freqtrade.resolvers import ExchangeResolver -from freqtrade.state import RunMode from freqtrade.worker import Worker from tests.conftest_trades import (mock_trade_1, mock_trade_2, mock_trade_3, mock_trade_4, mock_trade_5, mock_trade_6) @@ -326,6 +326,7 @@ def get_default_conf(testdatadir): "strategy_path": str(Path(__file__).parent / "strategy" / "strats"), "strategy": "DefaultStrategy", "internals": {}, + "export": "none", } return configuration @@ -1087,6 +1088,40 @@ def order_book_l2(): }) +@pytest.fixture +def order_book_l2_usd(): + return MagicMock(return_value={ + 'symbol': 'LTC/USDT', + 'bids': [ + [25.563, 49.269], + [25.562, 83.0], + [25.56, 106.0], + [25.559, 15.381], + [25.558, 29.299], + [25.557, 34.624], + [25.556, 10.0], + [25.555, 14.684], + [25.554, 45.91], + [25.553, 50.0] + ], + 'asks': [ + [25.566, 14.27], + [25.567, 48.484], + [25.568, 92.349], + [25.572, 31.48], + [25.573, 23.0], + [25.574, 20.0], + [25.575, 89.606], + [25.576, 262.016], + [25.577, 178.557], + [25.578, 78.614] + ], + 'timestamp': None, + 'datetime': None, + 'nonce': 2372149736 + }) + + @pytest.fixture def ohlcv_history_list(): return [ @@ -1913,7 +1948,7 @@ def saved_hyperopt_results_legacy(): @pytest.fixture def saved_hyperopt_results(): - return [ + hyperopt_res = [ { 'loss': 0.4366182531161519, 'params_dict': { @@ -2042,3 +2077,9 @@ def saved_hyperopt_results(): 'is_best': False } ] + + for res in hyperopt_res: + res['results_metrics']['holding_avg_s'] = res['results_metrics']['holding_avg' + ].total_seconds() + + return hyperopt_res diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 68960af1c..802fd4b12 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -1,5 +1,7 @@ # pragma pylint: disable=missing-docstring, C0103 import logging +from pathlib import Path +from shutil import copyfile import pytest @@ -11,7 +13,7 @@ from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_forma from freqtrade.data.history import (get_timerange, load_data, load_pair_history, validate_backtest_data) from tests.conftest import log_has, log_has_re -from tests.data.test_history import _backup_file, _clean_test_file +from tests.data.test_history import _clean_test_file def test_dataframe_correct_columns(result): @@ -251,17 +253,18 @@ def test_trades_dict_to_list(fetch_trades_result): assert t[6] == fetch_trades_result[i]['cost'] -def test_convert_trades_format(mocker, default_conf, testdatadir): - files = [{'old': testdatadir / "XRP_ETH-trades.json.gz", - 'new': testdatadir / "XRP_ETH-trades.json"}, - {'old': testdatadir / "XRP_OLD-trades.json.gz", - 'new': testdatadir / "XRP_OLD-trades.json"}, +def test_convert_trades_format(default_conf, testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) + files = [{'old': tmpdir1 / "XRP_ETH-trades.json.gz", + 'new': tmpdir1 / "XRP_ETH-trades.json"}, + {'old': tmpdir1 / "XRP_OLD-trades.json.gz", + 'new': tmpdir1 / "XRP_OLD-trades.json"}, ] for file in files: - _backup_file(file['old'], copy_file=True) + copyfile(testdatadir / file['old'].name, file['old']) assert not file['new'].exists() - default_conf['datadir'] = testdatadir + default_conf['datadir'] = tmpdir1 convert_trades_format(default_conf, convert_from='jsongz', convert_to='json', erase=False) @@ -284,14 +287,20 @@ def test_convert_trades_format(mocker, default_conf, testdatadir): file['new'].unlink() -def test_convert_ohlcv_format(mocker, default_conf, testdatadir): - file1 = testdatadir / "XRP_ETH-5m.json" - file1_new = testdatadir / "XRP_ETH-5m.json.gz" - file2 = testdatadir / "XRP_ETH-1m.json" - file2_new = testdatadir / "XRP_ETH-1m.json.gz" - _backup_file(file1, copy_file=True) - _backup_file(file2, copy_file=True) - default_conf['datadir'] = testdatadir +def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) + + file1_orig = testdatadir / "XRP_ETH-5m.json" + file1 = tmpdir1 / "XRP_ETH-5m.json" + file1_new = tmpdir1 / "XRP_ETH-5m.json.gz" + file2_orig = testdatadir / "XRP_ETH-1m.json" + file2 = tmpdir1 / "XRP_ETH-1m.json" + file2_new = tmpdir1 / "XRP_ETH-1m.json.gz" + + copyfile(file1_orig, file1) + copyfile(file2_orig, file2) + + default_conf['datadir'] = tmpdir1 default_conf['pairs'] = ['XRP_ETH'] default_conf['timeframes'] = ['1m', '5m'] @@ -317,10 +326,3 @@ def test_convert_ohlcv_format(mocker, default_conf, testdatadir): assert file2.exists() assert not file1_new.exists() assert not file2_new.exists() - - _clean_test_file(file1) - _clean_test_file(file2) - if file1_new.exists(): - file1_new.unlink() - if file2_new.exists(): - file2_new.unlink() diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index b87258c64..e43309743 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -5,9 +5,9 @@ import pytest from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums import RunMode from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.plugins.pairlistmanager import PairListManager -from freqtrade.state import RunMode from tests.conftest import get_patched_exchange diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 353cfc6f7..d203d0792 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -86,14 +86,12 @@ def test_load_data_7min_timeframe(mocker, caplog, default_conf, testdatadir) -> def test_load_data_1min_timeframe(ohlcv_history, mocker, caplog, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history) file = testdatadir / 'UNITTEST_BTC-1m.json' - _backup_file(file, copy_file=True) load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC']) assert file.is_file() assert not log_has( 'Download history data for pair: "UNITTEST/BTC", interval: 1m ' 'and store in None.', caplog ) - _clean_test_file(file) def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: @@ -112,17 +110,17 @@ def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, - default_conf, testdatadir) -> None: + default_conf, tmpdir) -> None: """ Test load_pair_history() with 1 min timeframe """ + tmpdir1 = Path(tmpdir) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list) exchange = get_patched_exchange(mocker, default_conf) - file = testdatadir / 'MEME_BTC-1m.json' + file = tmpdir1 / 'MEME_BTC-1m.json' - _backup_file(file) # do not download a new pair if refresh_pairs isn't set - load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC') + load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') assert not file.is_file() assert log_has( 'No history data for pair: "MEME/BTC", timeframe: 1m. ' @@ -130,15 +128,14 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, ) # download a new pair if refresh_pairs is set - refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'], + refresh_data(datadir=tmpdir1, timeframe='1m', pairs=['MEME/BTC'], exchange=exchange) - load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC') + load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') assert file.is_file() assert log_has_re( 'Download history data for pair: "MEME/BTC", timeframe: 1m ' 'and store in .*', caplog ) - _clean_test_file(file) def test_testdata_path(testdatadir) -> None: @@ -231,26 +228,22 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: assert start_ts is None -def test_download_pair_history(ohlcv_history_list, mocker, default_conf, testdatadir) -> None: +def test_download_pair_history(ohlcv_history_list, mocker, default_conf, tmpdir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list) exchange = get_patched_exchange(mocker, default_conf) - file1_1 = testdatadir / 'MEME_BTC-1m.json' - file1_5 = testdatadir / 'MEME_BTC-5m.json' - file2_1 = testdatadir / 'CFI_BTC-1m.json' - file2_5 = testdatadir / 'CFI_BTC-5m.json' - - _backup_file(file1_1) - _backup_file(file1_5) - _backup_file(file2_1) - _backup_file(file2_5) + tmpdir1 = Path(tmpdir) + file1_1 = tmpdir1 / 'MEME_BTC-1m.json' + file1_5 = tmpdir1 / 'MEME_BTC-5m.json' + file2_1 = tmpdir1 / 'CFI_BTC-1m.json' + file2_5 = tmpdir1 / 'CFI_BTC-5m.json' assert not file1_1.is_file() assert not file2_1.is_file() - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', timeframe='1m') - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='CFI/BTC', timeframe='1m') assert not exchange._pairs_last_refresh_time @@ -264,20 +257,16 @@ def test_download_pair_history(ohlcv_history_list, mocker, default_conf, testdat assert not file1_5.is_file() assert not file2_5.is_file() - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', timeframe='5m') - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='CFI/BTC', timeframe='5m') assert not exchange._pairs_last_refresh_time assert file1_5.is_file() assert file2_5.is_file() - # clean files freshly downloaded - _clean_test_file(file1_5) - _clean_test_file(file2_5) - def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: tick = [ @@ -294,24 +283,15 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: assert json_dump_mock.call_count == 2 -def test_download_backtesting_data_exception(ohlcv_history, mocker, caplog, - default_conf, testdatadir) -> None: +def test_download_backtesting_data_exception(mocker, caplog, default_conf, tmpdir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', side_effect=Exception('File Error')) - + tmpdir1 = Path(tmpdir) exchange = get_patched_exchange(mocker, default_conf) - file1_1 = testdatadir / 'MEME_BTC-1m.json' - file1_5 = testdatadir / 'MEME_BTC-5m.json' - _backup_file(file1_1) - _backup_file(file1_5) - - assert not _download_pair_history(datadir=testdatadir, exchange=exchange, + assert not _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', timeframe='1m') - # clean files freshly downloaded - _clean_test_file(file1_1) - _clean_test_file(file1_5) assert log_has('Failed to download history data for pair: "MEME/BTC", timeframe: 1m.', caplog) @@ -528,15 +508,15 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes assert log_has("Skipping pair XRP/ETH...", caplog) -def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog) -> None: - +def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog, + tmpdir) -> None: + tmpdir1 = Path(tmpdir) ght_mock = MagicMock(side_effect=lambda pair, *args, **kwargs: (pair, trades_history)) mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', ght_mock) exchange = get_patched_exchange(mocker, default_conf) - file1 = testdatadir / 'ETH_BTC-trades.json.gz' - data_handler = get_datahandler(testdatadir, data_format='jsongz') - _backup_file(file1) + file1 = tmpdir1 / 'ETH_BTC-trades.json.gz' + data_handler = get_datahandler(tmpdir1, data_format='jsongz') assert not file1.is_file() @@ -557,8 +537,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time2 - 5 assert ght_mock.call_args_list[0][1]['from_id'] is not None - # clean files freshly downloaded - _clean_test_file(file1) + file1.unlink() mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', MagicMock(side_effect=ValueError)) @@ -567,9 +546,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad pair='ETH/BTC') assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) - file2 = testdatadir / 'XRP_ETH-trades.json.gz' - - _backup_file(file2, True) + file2 = tmpdir1 / 'XRP_ETH-trades.json.gz' + copyfile(testdatadir / file2.name, file2) ght_mock.reset_mock() mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', @@ -589,38 +567,37 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad _clean_test_file(file2) -def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): - +def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog): + tmpdir1 = Path(tmpdir) pair = 'XRP/ETH' - file1 = testdatadir / 'XRP_ETH-1m.json' - file5 = testdatadir / 'XRP_ETH-5m.json' - # Compare downloaded dataset with converted dataset - dfbak_1m = load_pair_history(datadir=testdatadir, timeframe="1m", pair=pair) - dfbak_5m = load_pair_history(datadir=testdatadir, timeframe="5m", pair=pair) + file1 = tmpdir1 / 'XRP_ETH-1m.json' + file5 = tmpdir1 / 'XRP_ETH-5m.json' + filetrades = tmpdir1 / 'XRP_ETH-trades.json.gz' + copyfile(testdatadir / file1.name, file1) + copyfile(testdatadir / file5.name, file5) + copyfile(testdatadir / filetrades.name, filetrades) - _backup_file(file1, copy_file=True) - _backup_file(file5) + # Compare downloaded dataset with converted dataset + dfbak_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair) + dfbak_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair) tr = TimeRange.parse_timerange('20191011-20191012') convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'], - datadir=testdatadir, timerange=tr, erase=True) + datadir=tmpdir1, timerange=tr, erase=True) assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog) # Load new data - df_1m = load_pair_history(datadir=testdatadir, timeframe="1m", pair=pair) - df_5m = load_pair_history(datadir=testdatadir, timeframe="5m", pair=pair) + df_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair) + df_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair) assert df_1m.equals(dfbak_1m) assert df_5m.equals(dfbak_5m) - _clean_test_file(file1) - _clean_test_file(file5) - assert not log_has('Could not convert NoDatapair to OHLCV.', caplog) convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], - datadir=testdatadir, timerange=tr, erase=True) + datadir=tmpdir1, timerange=tr, erase=True) assert log_has('Could not convert NoDatapair to OHLCV.', caplog) @@ -752,15 +729,17 @@ def test_hdf5datahandler_trades_load(testdatadir): assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0 -def test_hdf5datahandler_trades_store(testdatadir): +def test_hdf5datahandler_trades_store(testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) dh = HDF5DataHandler(testdatadir) trades = dh.trades_load('XRP/ETH') - dh.trades_store('XRP/NEW', trades) - file = testdatadir / 'XRP_NEW-trades.h5' + dh1 = HDF5DataHandler(tmpdir1) + dh1.trades_store('XRP/NEW', trades) + file = tmpdir1 / 'XRP_NEW-trades.h5' assert file.is_file() # Load trades back - trades_new = dh.trades_load('XRP/NEW') + trades_new = dh1.trades_load('XRP/NEW') assert len(trades_new) == len(trades) assert trades[0][0] == trades_new[0][0] @@ -778,8 +757,6 @@ def test_hdf5datahandler_trades_store(testdatadir): assert trades[-1][5] == trades_new[-1][5] assert trades[-1][6] == trades_new[-1][6] - _clean_test_file(file) - def test_hdf5datahandler_trades_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) @@ -793,16 +770,18 @@ def test_hdf5datahandler_trades_purge(mocker, testdatadir): assert unlinkmock.call_count == 1 -def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir): +def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) dh = HDF5DataHandler(testdatadir) ohlcv = dh.ohlcv_load('UNITTEST/BTC', '5m') assert isinstance(ohlcv, DataFrame) assert len(ohlcv) > 0 - file = testdatadir / 'UNITTEST_NEW-5m.h5' + file = tmpdir1 / 'UNITTEST_NEW-5m.h5' assert not file.is_file() - dh.ohlcv_store('UNITTEST/NEW', '5m', ohlcv) + dh1 = HDF5DataHandler(tmpdir1) + dh1.ohlcv_store('UNITTEST/NEW', '5m', ohlcv) assert file.is_file() assert not ohlcv[ohlcv['date'] < '2018-01-15'].empty @@ -812,14 +791,12 @@ def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir): # Call private function to ensure timerange is filtered in hdf5 ohlcv = dh._ohlcv_load('UNITTEST/BTC', '5m', timerange) - ohlcv1 = dh._ohlcv_load('UNITTEST/NEW', '5m', timerange) + ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', '5m', timerange) assert len(ohlcv) == len(ohlcv1) assert ohlcv.equals(ohlcv1) assert ohlcv[ohlcv['date'] < '2018-01-15'].empty assert ohlcv[ohlcv['date'] > '2018-01-19'].empty - _clean_test_file(file) - # Try loading inexisting file ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', '5m') assert ohlcv.empty diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index c4620e1c7..0655b3a0f 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -12,8 +12,8 @@ from pandas import DataFrame, to_datetime from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo +from freqtrade.enums import SellType from freqtrade.exceptions import OperationalException -from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index b6b395802..ba9ec8bc7 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -11,7 +11,7 @@ import pytest from pandas import DataFrame from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException, - OperationalException, TemporaryError) + OperationalException, PricingError, TemporaryError) from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, calculate_backoff) @@ -947,6 +947,72 @@ def test_create_dry_run_order(default_conf, mocker, side, exchange_name): assert order["symbol"] == "ETH/BTC" +@pytest.mark.parametrize("side,startprice,endprice", [ + ("buy", 25.563, 25.566), + ("sell", 25.566, 25.563) +]) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice, + exchange_name, order_book_l2_usd): + default_conf['dry_run'] = True + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + fetch_l2_order_book=order_book_l2_usd, + ) + + order = exchange.create_dry_run_order( + pair='LTC/USDT', ordertype='limit', side=side, amount=1, rate=startprice) + assert order_book_l2_usd.call_count == 1 + assert 'id' in order + assert f'dry_run_{side}_' in order["id"] + assert order["side"] == side + assert order["type"] == "limit" + assert order["symbol"] == "LTC/USDT" + order_book_l2_usd.reset_mock() + + order_closed = exchange.fetch_dry_run_order(order['id']) + assert order_book_l2_usd.call_count == 1 + assert order_closed['status'] == 'open' + assert not order['fee'] + + order_book_l2_usd.reset_mock() + order_closed['price'] = endprice + + order_closed = exchange.fetch_dry_run_order(order['id']) + assert order_closed['status'] == 'closed' + assert order['fee'] + + +@pytest.mark.parametrize("side,amount,endprice", [ + ("buy", 1, 25.566), + ("buy", 100, 25.5672), # Requires interpolation + ("buy", 1000, 25.575), # More than orderbook return + ("sell", 1, 25.563), + ("sell", 100, 25.5625), # Requires interpolation + ("sell", 1000, 25.5555), # More than orderbook return +]) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_create_dry_run_order_market_fill(default_conf, mocker, side, amount, endprice, + exchange_name, order_book_l2_usd): + default_conf['dry_run'] = True + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + fetch_l2_order_book=order_book_l2_usd, + ) + + order = exchange.create_dry_run_order( + pair='LTC/USDT', ordertype='market', side=side, amount=amount, rate=25.5) + assert 'id' in order + assert f'dry_run_{side}_' in order["id"] + assert order["side"] == side + assert order["type"] == "market" + assert order["symbol"] == "LTC/USDT" + assert order['status'] == 'closed' + assert round(order["average"], 4) == round(endprice, 4) + + @pytest.mark.parametrize("side", [ ("buy"), ("sell") @@ -1684,6 +1750,152 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50) +@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [ + ('ask', 20, 19, 10, 0.0, 20), # Full ask side + ('ask', 20, 19, 10, 1.0, 10), # Full last side + ('ask', 20, 19, 10, 0.5, 15), # Between ask and last + ('ask', 20, 19, 10, 0.7, 13), # Between ask and last + ('ask', 20, 19, 10, 0.3, 17), # Between ask and last + ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask + ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask + ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask + ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask + ('ask', 4, 5, None, 1, 4), # last not available - uses ask + ('ask', 4, 5, None, 0, 4), # last not available - uses ask + ('bid', 21, 20, 10, 0.0, 20), # Full bid side + ('bid', 21, 20, 10, 1.0, 10), # Full last side + ('bid', 21, 20, 10, 0.5, 15), # Between bid and last + ('bid', 21, 20, 10, 0.7, 13), # Between bid and last + ('bid', 21, 20, 10, 0.3, 17), # Between bid and last + ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid + ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid + ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid + ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid + ('bid', 6, 5, None, 1, 5), # last not available - uses bid + ('bid', 6, 5, None, 0, 5), # last not available - uses bid +]) +def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, + last, last_ab, expected) -> None: + caplog.set_level(logging.DEBUG) + default_conf['bid_strategy']['ask_last_balance'] = last_ab + default_conf['bid_strategy']['price_side'] = side + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': ask, 'last': last, 'bid': bid}) + + assert exchange.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + assert exchange.get_buy_rate('ETH/BTC', False) == expected + assert log_has("Using cached buy rate for ETH/BTC.", caplog) + # Running a 2nd time with Refresh on! + caplog.clear() + assert exchange.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + +@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', [ + ('bid', 12.0, 11.0, 11.5, 0.0, 11.0), # full bid side + ('bid', 12.0, 11.0, 11.5, 1.0, 11.5), # full last side + ('bid', 12.0, 11.0, 11.5, 0.5, 11.25), # between bid and lat + ('bid', 12.0, 11.2, 10.5, 0.0, 11.2), # Last smaller than bid + ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid + ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid + ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), + ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side + ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side + ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat + ('ask', 12.2, 11.2, 10.5, 0.0, 12.2), # Last smaller than ask + ('ask', 12.0, 11.0, 10.5, 1.0, 12.0), # Last smaller than ask - uses ask + ('ask', 12.0, 11.2, 10.5, 0.5, 12.0), # Last smaller than ask - uses ask + ('ask', 10.0, 11.0, 11.0, 0.0, 10.0), + ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), + ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), + ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), +]) +def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, + last, last_ab, expected) -> None: + caplog.set_level(logging.DEBUG) + + default_conf['ask_strategy']['price_side'] = side + default_conf['ask_strategy']['bid_last_balance'] = last_ab + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': ask, 'bid': bid, 'last': last}) + pair = "ETH/BTC" + + # Test regular mode + exchange = get_patched_exchange(mocker, default_conf) + rate = exchange.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) + assert isinstance(rate, float) + assert rate == expected + # Use caching + rate = exchange.get_sell_rate(pair, False) + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + +@pytest.mark.parametrize('side,expected', [ + ('bid', 0.043936), # Value from order_book_l2 fiture - bids side + ('ask', 0.043949), # Value from order_book_l2 fiture - asks side +]) +def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2): + caplog.set_level(logging.DEBUG) + # Test orderbook mode + default_conf['ask_strategy']['price_side'] = side + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + pair = "ETH/BTC" + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) + exchange = get_patched_exchange(mocker, default_conf) + rate = exchange.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) + assert isinstance(rate, float) + assert rate == expected + rate = exchange.get_sell_rate(pair, False) + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + +def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog): + # Test orderbook mode + default_conf['ask_strategy']['price_side'] = 'ask' + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + pair = "ETH/BTC" + # Test What happens if the exchange returns an empty orderbook. + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', + return_value={'bids': [[]], 'asks': [[]]}) + exchange = get_patched_exchange(mocker, default_conf) + with pytest.raises(PricingError): + exchange.get_sell_rate(pair, True) + assert log_has("Sell Price at location from orderbook could not be determined.", caplog) + + +def test_get_sell_rate_exception(default_conf, mocker, caplog): + # Ticker on one side can be empty in certain circumstances. + default_conf['ask_strategy']['price_side'] = 'ask' + pair = "ETH/BTC" + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': None, 'bid': 0.12, 'last': None}) + exchange = get_patched_exchange(mocker, default_conf) + with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): + exchange.get_sell_rate(pair, True) + + exchange._config['ask_strategy']['price_side'] = 'bid' + assert exchange.get_sell_rate(pair, True) == 0.12 + # Reverse sides + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': 0.13, 'bid': None, 'last': None}) + with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): + exchange.get_sell_rate(pair, True) + + exchange._config['ask_strategy']['price_side'] = 'ask' + assert exchange.get_sell_rate(pair, True) == 0.13 + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: @@ -1971,6 +2183,7 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange def test_cancel_order_dry_run(default_conf, mocker, exchange_name): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=True) assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {} assert exchange.cancel_stoploss_order(order_id='123', pair='TKN/BTC') == {} @@ -2125,8 +2338,9 @@ def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_fetch_order(default_conf, mocker, exchange_name): +def test_fetch_order(default_conf, mocker, exchange_name, caplog): default_conf['dry_run'] = True + default_conf['exchange']['log_responses'] = True order = MagicMock() order.myid = 123 exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) @@ -2141,6 +2355,7 @@ def test_fetch_order(default_conf, mocker, exchange_name): api_mock.fetch_order = MagicMock(return_value=456) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) assert exchange.fetch_order('X', 'TKN/BTC') == 456 + assert log_has("API fetch_order: 456", caplog) with pytest.raises(InvalidOrderException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) diff --git a/tests/exchange/test_ftx.py b/tests/exchange/test_ftx.py index 63d99acdf..3794bb79c 100644 --- a/tests/exchange/test_ftx.py +++ b/tests/exchange/test_ftx.py @@ -125,7 +125,7 @@ def test_stoploss_adjust_ftx(mocker, default_conf): assert not exchange.stoploss_adjust(1501, order) -def test_fetch_stoploss_order(default_conf, mocker): +def test_fetch_stoploss_order(default_conf, mocker, limit_sell_order): default_conf['dry_run'] = True order = MagicMock() order.myid = 123 @@ -147,6 +147,17 @@ def test_fetch_stoploss_order(default_conf, mocker): with pytest.raises(InvalidOrderException, match=r"Could not get stoploss order for id X"): exchange.fetch_stoploss_order('X', 'TKN/BTC')['status'] + api_mock.fetch_orders = MagicMock(return_value=[{'id': 'X', 'status': 'closed'}]) + api_mock.fetch_order = MagicMock(return_value=limit_sell_order) + + resp = exchange.fetch_stoploss_order('X', 'TKN/BTC') + assert resp + assert api_mock.fetch_order.call_count == 1 + assert resp['id_stop'] == 'mocked_limit_sell' + assert resp['id'] == 'X' + assert resp['type'] == 'stop' + assert resp['status_stop'] == 'triggered' + with pytest.raises(InvalidOrderException): api_mock.fetch_orders = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id='ftx') @@ -165,8 +176,8 @@ def test_get_order_id(mocker, default_conf): 'type': STOPLOSS_ORDERTYPE, 'price': 1500, 'id': '1111', + 'id_stop': '1234', 'info': { - 'orderId': '1234' } } assert exchange.get_order_id_conditional(order) == '1234' @@ -175,8 +186,8 @@ def test_get_order_id(mocker, default_conf): 'type': 'limit', 'price': 1500, 'id': '1111', + 'id_stop': '1234', 'info': { - 'orderId': '1234' } } assert exchange.get_order_id_conditional(order) == '1111' diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index 306850ff6..7cca2b1a8 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -3,8 +3,8 @@ from typing import Dict, List, NamedTuple, Optional import arrow from pandas import DataFrame +from freqtrade.enums import SellType from freqtrade.exchange import timeframe_to_minutes -from freqtrade.strategy.interface import SellType tests_start_time = arrow.get(2018, 10, 3) diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index 11b4674f3..a7fd238d1 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -5,9 +5,8 @@ from pathlib import Path import pandas as pd import pytest +from freqtrade.enums import RunMode, SellType from freqtrade.optimize.hyperopt import Hyperopt -from freqtrade.state import RunMode -from freqtrade.strategy.interface import SellType from tests.conftest import patch_exchange diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index c35a083ec..488425323 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -4,8 +4,8 @@ import logging import pytest from freqtrade.data.history import get_timerange +from freqtrade.enums import SellType from freqtrade.optimize.backtesting import Backtesting -from freqtrade.strategy.interface import SellType from tests.conftest import patch_exchange from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset, tests_timeframe) @@ -457,6 +457,50 @@ tc28 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)] ) +# Test 29: trailing_stop should be triggered by low of next candle, without adjusting stoploss using +# high of stoploss candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc29 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Triggers trailing-stoploss + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.02, trailing_stop=True, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=2)] +) + +# Test 30: trailing_stop should be triggered immediately on trade open candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc30 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) and stop + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.01, trailing_stop=True, + trailing_stop_positive=0.01, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=1)] +) + +# Test 31: trailing_stop should be triggered immediately on trade open candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc31 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) and stop + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=0.01, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.02, + trailing_stop_positive=0.01, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=1)] +) + TESTS = [ tc0, tc1, @@ -487,6 +531,9 @@ TESTS = [ tc26, tc27, tc28, + tc29, + tc30, + tc31, ] diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 632d458ce..60bd82d71 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -16,12 +16,11 @@ from freqtrade.data.btanalysis import BT_DATA_COLUMNS, evaluate_result_multi from freqtrade.data.converter import clean_ohlcv_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange +from freqtrade.enums import RunMode, SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade from freqtrade.resolvers import StrategyResolver -from freqtrade.state import RunMode -from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -156,6 +155,7 @@ def test_setup_optimize_configuration_without_arguments(mocker, default_conf, ca 'backtesting', '--config', 'config.json', '--strategy', 'DefaultStrategy', + '--export', 'none' ] config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) @@ -173,7 +173,8 @@ def test_setup_optimize_configuration_without_arguments(mocker, default_conf, ca assert not log_has('Parameter --enable-position-stacking detected ...', caplog) assert 'timerange' not in config - assert 'export' not in config + assert 'export' in config + assert config['export'] == 'none' assert 'runmode' in config assert config['runmode'] == RunMode.BACKTEST @@ -194,7 +195,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--enable-position-stacking', '--disable-max-market-positions', '--timerange', ':100', - '--export', '/bar/foo', '--export-filename', 'foo_bar.json', '--fee', '0', ] @@ -224,7 +224,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) assert 'export' in config - assert log_has('Parameter --export detected: {} ...'.format(config['export']), caplog) assert 'exportfilename' in config assert isinstance(config['exportfilename'], Path) assert log_has('Storing backtest results to {} ...'.format(config['exportfilename']), caplog) @@ -396,7 +395,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> default_conf['timeframe'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' default_conf['timerange'] = '20180101-20180102' backtesting = Backtesting(default_conf) @@ -417,7 +416,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> default_conf['timeframe'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' default_conf['timerange'] = '20180101-20180102' with pytest.raises(OperationalException, match='No pair in whitelist.'): @@ -441,7 +440,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti default_conf['ticker_interval'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' # Use stoploss from strategy del default_conf['stoploss'] default_conf['timerange'] = '20180101-20180102' diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index 188b4aa5f..6818a573b 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -4,8 +4,8 @@ from unittest.mock import MagicMock from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge +from freqtrade.enums import RunMode from freqtrade.optimize.edge_cli import EdgeCli -from freqtrade.state import RunMode from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index d7eb8bf67..10e99395d 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import locale import logging import re from datetime import datetime @@ -14,6 +13,7 @@ from filelock import Timeout from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt from freqtrade.data.history import load_data +from freqtrade.enums import RunMode, SellType from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt import Hyperopt from freqtrade.optimize.hyperopt_auto import HyperOptAuto @@ -21,9 +21,7 @@ from freqtrade.optimize.hyperopt_tools import HyperoptTools from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.optimize.space import SKDecimal from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver -from freqtrade.state import RunMode from freqtrade.strategy.hyper import IntParameter -from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -640,9 +638,9 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'loss': 1.9147239021396234, 'results_explanation': (' 4 trades. 4/0/0 Wins/Draws/Losses. ' 'Avg profit 0.77%. Median profit 0.71%. Total profit ' - '0.00003100 BTC ( 0.00\N{GREEK CAPITAL LETTER SIGMA}%). ' + '0.00003100 BTC ( 0.00%). ' 'Avg duration 0:50:00 min.' - ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8'), + ), 'params_details': {'buy': {'adx-enabled': False, 'adx-value': 0, 'fastd-enabled': True, @@ -1070,7 +1068,7 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No hyperopt.start() -def test_print_epoch_details(capsys): +def test_show_epoch_details(capsys): test_result = { 'params_details': { 'trailing': { @@ -1092,7 +1090,7 @@ def test_print_epoch_details(capsys): 'is_best': True } - HyperoptTools.print_epoch_details(test_result, 5, False, no_header=True) + HyperoptTools.show_epoch_details(test_result, 5, False, no_header=True) captured = capsys.readouterr() assert '# Trailing stop:' in captured.out # re.match(r"Pairs for .*", captured.out) diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index 73feeb007..ea0caac04 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -149,9 +149,9 @@ def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_result def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: results_over = hyperopt_results.copy() - results_over['profit_ratio'] = hyperopt_results['profit_ratio'] * 2 + results_over['profit_abs'] = hyperopt_results['profit_abs'] * 2 results_under = hyperopt_results.copy() - results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 + results_under['profit_abs'] = hyperopt_results['profit_abs'] / 2 default_conf.update({'hyperopt_loss': 'OnlyProfitHyperOptLoss'}) hl = HyperOptLossResolver.load_hyperoptloss(default_conf) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index f9dac3397..3f31efb95 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -12,6 +12,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN from freqtrade.data import history from freqtrade.data.btanalysis import get_latest_backtest_filename, load_backtest_data from freqtrade.edge import PairInfo +from freqtrade.enums import SellType from freqtrade.optimize.optimize_reports import (generate_backtest_stats, generate_daily_stats, generate_edge_table, generate_pair_metrics, generate_sell_reason_stats, @@ -20,7 +21,6 @@ from freqtrade.optimize.optimize_reports import (generate_backtest_stats, genera text_table_bt_results, text_table_sell_reason, text_table_strategy) from freqtrade.resolvers.strategy_resolver import StrategyResolver -from freqtrade.strategy.interface import SellType from tests.data.test_history import _backup_file, _clean_test_file @@ -51,7 +51,7 @@ def test_text_table_bt_results(): assert text_table_bt_results(pair_results, stake_currency='BTC') == result_str -def test_generate_backtest_stats(default_conf, testdatadir): +def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): default_conf.update({'strategy': 'DefaultStrategy'}) StrategyResolver.load_strategy(default_conf) @@ -148,8 +148,8 @@ def test_generate_backtest_stats(default_conf, testdatadir): assert strat_stats['pairlist'] == ['UNITTEST/BTC'] # Test storing stats - filename = Path(testdatadir / 'btresult.json') - filename_last = Path(testdatadir / LAST_BT_RESULT_FN) + filename = Path(tmpdir / 'btresult.json') + filename_last = Path(tmpdir / LAST_BT_RESULT_FN) _backup_file(filename_last, copy_file=True) assert not filename.is_file() @@ -159,7 +159,7 @@ def test_generate_backtest_stats(default_conf, testdatadir): last_fn = get_latest_backtest_filename(filename_last.parent) assert re.match(r"btresult-.*\.json", last_fn) - filename1 = (testdatadir / last_fn) + filename1 = Path(tmpdir / last_fn) assert filename1.is_file() content = filename1.read_text() assert 'max_drawdown' in content diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 5e2274ce3..ae8f6e958 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -75,7 +75,7 @@ def whitelist_conf_agefilter(default_conf): "method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", - "refresh_period": 0, + "refresh_period": -1, }, { "method": "AgeFilter", @@ -687,7 +687,6 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o freqtrade.pairlists.refresh_pairlist() assert len(freqtrade.pairlists.whitelist) == 3 assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 - # freqtrade.config['exchange']['pair_whitelist'].append('HOT/BTC') previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count freqtrade.pairlists.refresh_pairlist() diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index a39301145..9ec47dade 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -4,9 +4,9 @@ from datetime import datetime, timedelta import pytest from freqtrade import constants +from freqtrade.enums import SellType from freqtrade.persistence import PairLocks, Trade from freqtrade.plugins.protectionmanager import ProtectionManager -from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has_re @@ -70,8 +70,7 @@ def test_protectionmanager(mocker, default_conf): ]) def test_protections_init(mocker, default_conf, timeframe, expected, protconf): default_conf['timeframe'] = timeframe - default_conf['protections'] = protconf - man = ProtectionManager(default_conf) + man = ProtectionManager(default_conf, protconf) assert len(man._protection_handlers) == len(protconf) assert man._protection_handlers[0]._lookback_period == expected[0] assert man._protection_handlers[0]._stop_duration == expected[1] diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index b005fb105..cc29dc157 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -8,12 +8,12 @@ import pytest from numpy import isnan from freqtrade.edge import PairInfo +from freqtrade.enums import State from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError from freqtrade.persistence import Trade from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter -from freqtrade.state import State from tests.conftest import create_mock_trades, get_patched_freqtradebot, patch_get_signal @@ -109,7 +109,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'exchange': 'binance', } - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) @@ -217,7 +217,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert '-0.41% (-0.06)' == result[0][3] assert '-0.06' == f'{fiat_profit_sum:.2f}' - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) result, headers, fiat_profit_sum = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] @@ -422,18 +422,18 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] == '0:00:00' + assert stats['avg_duration'] in ('0:00:00', '0:00:01') assert stats['best_pair'] == 'ETH/BTC' assert prec_satoshi(stats['best_rate'], 6.2) # Test non-available pair - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] == '0:00:00' + assert stats['avg_duration'] in ('0:00:00', '0:00:01') assert stats['best_pair'] == 'ETH/BTC' assert prec_satoshi(stats['best_rate'], 6.2) assert isnan(stats['profit_all_coin']) @@ -679,6 +679,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: 'filled': 0.0, } ), + _is_dry_limit_order_filled=MagicMock(return_value=True), get_fee=fee, ) mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=1000) @@ -703,8 +704,8 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: assert msg == {'result': 'Created sell orders for all open trades.'} freqtradebot.enter_positions() - msg = rpc._rpc_forcesell('1') - assert msg == {'result': 'Created sell order for trade 1.'} + msg = rpc._rpc_forcesell('2') + assert msg == {'result': 'Created sell order for trade 2.'} freqtradebot.state = State.STOPPED with pytest.raises(RPCException, match=r'.*trader is not running*'): @@ -715,9 +716,11 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 + mocker.patch( + 'freqtrade.exchange.Exchange._is_dry_limit_order_filled', MagicMock(return_value=False)) freqtradebot.enter_positions() # make an limit-buy open trade - trade = Trade.query.filter(Trade.id == '1').first() + trade = Trade.query.filter(Trade.id == '3').first() filled_amount = trade.amount / 2 # Fetch order - it's open first, and closed after cancel_order is called. mocker.patch( @@ -738,7 +741,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called # and trade amount is updated - rpc._rpc_forcesell('1') + rpc._rpc_forcesell('3') assert cancel_order_mock.call_count == 1 assert trade.amount == filled_amount @@ -766,8 +769,8 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: } ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called - msg = rpc._rpc_forcesell('2') - assert msg == {'result': 'Created sell order for trade 2.'} + msg = rpc._rpc_forcesell('4') + assert msg == {'result': 'Created sell order for trade 4.'} assert cancel_order_mock.call_count == 2 assert trade.amount == amount diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 1a66b2e81..b8dd112c9 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -15,6 +15,7 @@ from numpy import isnan from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ +from freqtrade.enums import RunMode, State from freqtrade.exceptions import ExchangeError from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade @@ -22,7 +23,6 @@ from freqtrade.rpc import RPC from freqtrade.rpc.api_server import ApiServer from freqtrade.rpc.api_server.api_auth import create_token, get_user_from_token from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer -from freqtrade.state import RunMode, State from tests.conftest import (create_mock_trades, get_mock_coro, get_patched_freqtradebot, log_has, log_has_re, patch_get_signal) @@ -834,7 +834,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'exchange': 'binance', } - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) rc = client_get(client, f"{BASE_URI}/status") @@ -996,7 +996,8 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): get_balances=MagicMock(return_value=ticker), fetch_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) + markets=PropertyMock(return_value=markets), + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_get_signal(ftbot, (True, False)) diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index 69a757fcf..918022386 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -3,7 +3,8 @@ import logging import time from unittest.mock import MagicMock -from freqtrade.rpc import RPCManager, RPCMessageType +from freqtrade.enums import RPCMessageType +from freqtrade.rpc import RPCManager from tests.conftest import get_patched_freqtradebot, log_has diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index e640f2dff..782ae69c6 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -2,6 +2,7 @@ # pragma pylint: disable=protected-access, unused-argument, invalid-name # pragma pylint: disable=too-many-lines, too-many-arguments +import logging import re from datetime import datetime from functools import reduce @@ -12,21 +13,20 @@ from unittest.mock import ANY, MagicMock import arrow import pytest from telegram import Chat, Message, ReplyKeyboardMarkup, Update -from telegram.error import NetworkError +from telegram.error import BadRequest, NetworkError, TelegramError from freqtrade import __version__ from freqtrade.constants import CANCEL_REASON from freqtrade.edge import PairInfo +from freqtrade.enums import RPCMessageType, RunMode, SellType, State from freqtrade.exceptions import OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.loggers import setup_logging from freqtrade.persistence import PairLocks, Trade -from freqtrade.rpc import RPC, RPCMessageType +from freqtrade.rpc import RPC from freqtrade.rpc.telegram import Telegram, authorized_only -from freqtrade.state import RunMode, State -from freqtrade.strategy.interface import SellType -from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, patch_exchange, - patch_get_signal, patch_whitelist) +from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, log_has_re, + patch_exchange, patch_get_signal, patch_whitelist) class DummyCls(Telegram): @@ -55,14 +55,6 @@ class DummyCls(Telegram): raise Exception('test') -def get_telegram_testobject_with_inline(mocker, default_conf, mock=True, ftbot=None): - inline_msg_mock = MagicMock() - telegram, ftbot, msg_mock = get_telegram_testobject(mocker, default_conf) - mocker.patch('freqtrade.rpc.telegram.Telegram._send_inline_msg', inline_msg_mock) - - return telegram, ftbot, msg_mock, inline_msg_mock - - def get_telegram_testobject(mocker, default_conf, mock=True, ftbot=None): msg_mock = MagicMock() if mock: @@ -121,7 +113,7 @@ def test_cleanup(default_conf, mocker, ) -> None: def test_authorized_only(default_conf, mocker, caplog, update) -> None: patch_exchange(mocker) - + caplog.set_level(logging.DEBUG) default_conf['telegram']['enabled'] = False bot = FreqtradeBot(default_conf) rpc = RPC(bot) @@ -137,6 +129,7 @@ def test_authorized_only(default_conf, mocker, caplog, update) -> None: def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: patch_exchange(mocker) + caplog.set_level(logging.DEBUG) chat = Chat(0xdeadbeef, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), datetime.utcnow(), chat) @@ -225,6 +218,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=True), ) status_table = MagicMock() mocker.patch.multiple( @@ -451,8 +445,10 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) - - telegram._profit(update=update, context=MagicMock()) + context = MagicMock() + # Test with invalid 2nd argument (should silently pass) + context.args = ["aaa"] + telegram._profit(update=update, context=context) assert msg_mock.call_count == 1 assert 'No closed trade' in msg_mock.call_args_list[-1][0][0] assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] @@ -671,6 +667,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=True), ) freqtradebot = FreqtradeBot(default_conf) @@ -729,6 +726,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=True), ) freqtradebot = FreqtradeBot(default_conf) @@ -789,6 +787,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=True), ) default_conf['max_open_trades'] = 4 freqtradebot = FreqtradeBot(default_conf) @@ -805,9 +804,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None context.args = ["all"] telegram._forcesell(update=update, context=context) - # Called for each trade 4 times - assert msg_mock.call_count == 12 - msg = msg_mock.call_args_list[2][0][0] + # Called for each trade 2 times + assert msg_mock.call_count == 8 + msg = msg_mock.call_args_list[1][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, @@ -917,8 +916,8 @@ def test_forcebuy_no_pair(default_conf, update, mocker) -> None: fbuy_mock = MagicMock(return_value=None) mocker.patch('freqtrade.rpc.RPC._rpc_forcebuy', fbuy_mock) - telegram, freqtradebot, _, inline_msg_mock = get_telegram_testobject_with_inline(mocker, - default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) context = MagicMock() @@ -926,10 +925,10 @@ def test_forcebuy_no_pair(default_conf, update, mocker) -> None: telegram._forcebuy(update=update, context=context) assert fbuy_mock.call_count == 0 - assert inline_msg_mock.call_count == 1 - assert inline_msg_mock.call_args_list[0][0][0] == 'Which pair?' - assert inline_msg_mock.call_args_list[0][1]['callback_query_handler'] == 'forcebuy' - keyboard = inline_msg_mock.call_args_list[0][1]['keyboard'] + assert msg_mock.call_count == 1 + assert msg_mock.call_args_list[0][1]['msg'] == 'Which pair?' + # assert msg_mock.call_args_list[0][1]['callback_query_handler'] == 'forcebuy' + keyboard = msg_mock.call_args_list[0][1]['keyboard'] assert reduce(lambda acc, x: acc + len(x), keyboard, 0) == 4 update = MagicMock() update.callback_query = MagicMock() @@ -1350,13 +1349,14 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] \ == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n' + '*Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' + '*Sell Reason:* `stop_loss`\n' + '*Duration:* `1:00:00 (60.0 min)`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' - '*Close Rate:* `0.00003201`\n' - '*Sell Reason:* `stop_loss`\n' - '*Duration:* `1:00:00 (60.0 min)`\n' - '*Profit:* `-57.41%` `(loss: -0.05746268 ETH / -24.812 USD)`') + '*Close Rate:* `0.00003201`' + ) msg_mock.reset_mock() telegram.send_msg({ @@ -1379,13 +1379,14 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] \ == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n' + '*Profit:* `-57.41%`\n' + '*Sell Reason:* `stop_loss`\n' + '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' - '*Close Rate:* `0.00003201`\n' - '*Sell Reason:* `stop_loss`\n' - '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n' - '*Profit:* `-57.41%`') + '*Close Rate:* `0.00003201`' + ) # Reset singleton function to avoid random breaks telegram._rpc._fiat_converter.convert_amount = old_convamount @@ -1537,13 +1538,14 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n' + '*Profit:* `-57.41%`\n' + '*Sell Reason:* `stop_loss`\n' + '*Duration:* `2:35:03 (155.1 min)`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' - '*Close Rate:* `0.00003201`\n' - '*Sell Reason:* `stop_loss`\n' - '*Duration:* `2:35:03 (155.1 min)`\n' - '*Profit:* `-57.41%`') + '*Close Rate:* `0.00003201`' + ) @pytest.mark.parametrize('msg,expected', [ @@ -1563,7 +1565,7 @@ def test__sell_emoji(default_conf, mocker, msg, expected): assert telegram._get_sell_emoji(msg) == expected -def test__send_msg(default_conf, mocker) -> None: +def test_telegram__send_msg(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) bot = MagicMock() telegram, _, _ = get_telegram_testobject(mocker, default_conf, mock=False) @@ -1574,6 +1576,28 @@ def test__send_msg(default_conf, mocker) -> None: telegram._send_msg('test') assert len(bot.method_calls) == 1 + # Test update + query = MagicMock() + telegram._send_msg('test', callback_path="DeadBeef", query=query, reload_able=True) + edit_message_text = telegram._updater.bot.edit_message_text + assert edit_message_text.call_count == 1 + assert "Updated: " in edit_message_text.call_args_list[0][1]['text'] + + telegram._updater.bot.edit_message_text = MagicMock(side_effect=BadRequest("not modified")) + telegram._send_msg('test', callback_path="DeadBeef", query=query) + assert telegram._updater.bot.edit_message_text.call_count == 1 + assert not log_has_re(r"TelegramError: .*", caplog) + + telegram._updater.bot.edit_message_text = MagicMock(side_effect=BadRequest("")) + telegram._send_msg('test2', callback_path="DeadBeef", query=query) + assert telegram._updater.bot.edit_message_text.call_count == 1 + assert log_has_re(r"TelegramError: .*", caplog) + + telegram._updater.bot.edit_message_text = MagicMock(side_effect=TelegramError("DeadBEEF")) + telegram._send_msg('test3', callback_path="DeadBeef", query=query) + + assert log_has_re(r"TelegramError: DeadBEEF! Giving up.*", caplog) + def test__send_msg_network_error(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) @@ -1604,7 +1628,7 @@ def test__send_msg_keyboard(default_conf, mocker, caplog) -> None: ['/count', '/start', '/stop', '/help']] default_keyboard = ReplyKeyboardMarkup(default_keys_list) - custom_keys_list = [['/daily', '/stats', '/balance', '/profit'], + custom_keys_list = [['/daily', '/stats', '/balance', '/profit', '/profit 5'], ['/count', '/start', '/reload_config', '/help']] custom_keyboard = ReplyKeyboardMarkup(custom_keys_list) @@ -1638,5 +1662,5 @@ def test__send_msg_keyboard(default_conf, mocker, caplog) -> None: used_keyboard = bot.send_message.call_args[1]['reply_markup'] assert used_keyboard == custom_keyboard assert log_has("using custom keyboard from config.json: " - "[['/daily', '/stats', '/balance', '/profit'], ['/count', " + "[['/daily', '/stats', '/balance', '/profit', '/profit 5'], ['/count', " "'/start', '/reload_config', '/help']]", caplog) diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 0560f8d53..04e63a3be 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -5,9 +5,9 @@ from unittest.mock import MagicMock import pytest from requests import RequestException -from freqtrade.rpc import RPC, RPCMessageType +from freqtrade.enums import RPCMessageType, SellType +from freqtrade.rpc import RPC from freqtrade.rpc.webhook import Webhook -from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index ded396779..04d12a51f 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -10,12 +10,13 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import load_data +from freqtrade.enums import SellType from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.persistence import PairLocks, Trade from freqtrade.resolvers import StrategyResolver from freqtrade.strategy.hyper import (BaseParameter, CategoricalParameter, DecimalParameter, IntParameter, RealParameter) -from freqtrade.strategy.interface import SellCheckTuple, SellType +from freqtrade.strategy.interface import SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from tests.conftest import log_has, log_has_re @@ -152,6 +153,8 @@ def test_assert_df_raise(mocker, caplog, ohlcv_history): def test_assert_df(ohlcv_history, caplog): df_len = len(ohlcv_history) - 1 + ohlcv_history.loc[:, 'buy'] = 0 + ohlcv_history.loc[:, 'sell'] = 0 # Ensure it's running when passed correctly _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[df_len, 'date']) @@ -169,6 +172,18 @@ def test_assert_df(ohlcv_history, caplog): match=r"Dataframe returned from strategy.*last date\."): _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date']) + with pytest.raises(StrategyError, + match=r"No dataframe returned \(return statement missing\?\)."): + _STRATEGY.assert_df(None, len(ohlcv_history), + ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date']) + with pytest.raises(StrategyError, + match="Buy column not set"): + _STRATEGY.assert_df(ohlcv_history.drop('buy', axis=1), len(ohlcv_history), + ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date']) + with pytest.raises(StrategyError, + match="Sell column not set"): + _STRATEGY.assert_df(ohlcv_history.drop('sell', axis=1), len(ohlcv_history), + ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date']) _STRATEGY.disable_dataframe_checks = True caplog.clear() @@ -667,8 +682,13 @@ def test_auto_hyperopt_interface(default_conf): # Parameter is disabled - so value from sell_param dict will NOT be used. assert strategy.sell_minusdi.value == 0.5 + all_params = strategy.detect_all_parameters() + assert isinstance(all_params, dict) + assert len(all_params['buy']) == 2 + assert len(all_params['sell']) == 2 + assert all_params['count'] == 4 - strategy.sell_rsi = IntParameter([0, 10], default=5, space='buy') + strategy.__class__.sell_rsi = IntParameter([0, 10], default=5, space='buy') with pytest.raises(OperationalException, match=r"Inconclusive parameter.*"): - [x for x in strategy._detect_parameters('sell')] + [x for x in strategy.detect_parameters('sell')] diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 965c3d37b..bd192ecb5 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -129,13 +129,16 @@ def test_strategy_override_minimal_roi(caplog, default_conf): default_conf.update({ 'strategy': 'DefaultStrategy', 'minimal_roi': { + "20": 0.1, "0": 0.5 } }) strategy = StrategyResolver.load_strategy(default_conf) assert strategy.minimal_roi[0] == 0.5 - assert log_has("Override strategy 'minimal_roi' with value in config file: {'0': 0.5}.", caplog) + assert log_has( + "Override strategy 'minimal_roi' with value in config file: {'20': 0.1, '0': 0.5}.", + caplog) def test_strategy_override_stoploss(caplog, default_conf): diff --git a/tests/test_arguments.py b/tests/test_arguments.py index 60c2cfbac..0d81dea28 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -186,18 +186,22 @@ def test_plot_dataframe_options() -> None: assert pargs['pairs'] == ['UNITTEST/BTC'] -def test_plot_profit_options() -> None: +@pytest.mark.parametrize('auto_open_arg', [True, False]) +def test_plot_profit_options(auto_open_arg: bool) -> None: args = [ 'plot-profit', '-p', 'UNITTEST/BTC', '--trade-source', 'DB', '--db-url', 'sqlite:///whatever.sqlite', ] + if auto_open_arg: + args.append('--auto-open') pargs = Arguments(args).get_parsed_arg() assert pargs['trade_source'] == 'DB' assert pargs['pairs'] == ['UNITTEST/BTC'] assert pargs['db_url'] == 'sqlite:///whatever.sqlite' + assert pargs['plot_auto_open'] == auto_open_arg def test_config_notallowed(mocker) -> None: diff --git a/tests/test_configuration.py b/tests/test_configuration.py index b2c883108..c5d0cd908 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -18,11 +18,11 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti process_deprecated_setting, process_removed_setting, process_temporary_deprecated_settings) -from freqtrade.configuration.load_config import load_config_file, log_config_error_range +from freqtrade.configuration.load_config import load_config_file, load_file, log_config_error_range from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.loggers import _set_loggers, setup_logging, setup_logging_pre -from freqtrade.state import RunMode from tests.conftest import log_has, log_has_re, patched_configuration_load_config_file @@ -88,6 +88,24 @@ def test_load_config_file_error_range(default_conf, mocker, caplog) -> None: '"stake_amount": .001, "fiat_display_currency": "USD", ' '"timeframe": "5m", "dry_run": true, "cance') + filedata = json.dumps(default_conf, indent=2).replace( + '"stake_amount": 0.001,', '"stake_amount": .001,') + mocker.patch.object(Path, "read_text", MagicMock(return_value=filedata)) + + x = log_config_error_range('somefile', 'Parse error at offset 4: Invalid value.') + assert isinstance(x, str) + assert (x == ' "max_open_trades": 1,\n "stake_currency": "BTC",\n' + ' "stake_amount": .001,') + + x = log_config_error_range('-', '') + assert x == '' + + +def test_load_file_error(tmpdir): + testpath = Path(tmpdir) / 'config.json' + with pytest.raises(OperationalException, match=r"File .* not found!"): + load_file(testpath) + def test__args_to_config(caplog): @@ -407,7 +425,6 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert not log_has('Parameter --enable-position-stacking detected ...', caplog) assert 'timerange' not in config - assert 'export' not in config def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: @@ -430,7 +447,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--enable-position-stacking', '--disable-max-market-positions', '--timerange', ':100', - '--export', '/bar/foo', + '--export', 'trades', '--stake-amount', 'unlimited' ] @@ -478,7 +495,7 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non 'backtesting', '--config', 'config.json', '--ticker-interval', '1m', - '--export', '/bar/foo', + '--export', 'trades', '--strategy-list', 'DefaultStrategy', 'TestStrategy' diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 4d9284a2f..aac8d0c4c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -11,15 +11,14 @@ import arrow import pytest from freqtrade.constants import CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT +from freqtrade.enums import RPCMessageType, RunMode, SellType, State from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, TemporaryError) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock -from freqtrade.rpc import RPCMessageType -from freqtrade.state import RunMode, State -from freqtrade.strategy.interface import SellCheckTuple, SellType +from freqtrade.strategy.interface import SellCheckTuple from freqtrade.worker import Worker from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal, @@ -305,6 +304,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -334,6 +334,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) # Save state of current whitelist @@ -751,50 +752,6 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: assert ("ETH/BTC", default_conf["timeframe"]) in refresh_mock.call_args[0][0] -@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [ - ('ask', 20, 19, 10, 0.0, 20), # Full ask side - ('ask', 20, 19, 10, 1.0, 10), # Full last side - ('ask', 20, 19, 10, 0.5, 15), # Between ask and last - ('ask', 20, 19, 10, 0.7, 13), # Between ask and last - ('ask', 20, 19, 10, 0.3, 17), # Between ask and last - ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask - ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask - ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask - ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask - ('ask', 4, 5, None, 1, 4), # last not available - uses ask - ('ask', 4, 5, None, 0, 4), # last not available - uses ask - ('bid', 21, 20, 10, 0.0, 20), # Full bid side - ('bid', 21, 20, 10, 1.0, 10), # Full last side - ('bid', 21, 20, 10, 0.5, 15), # Between bid and last - ('bid', 21, 20, 10, 0.7, 13), # Between bid and last - ('bid', 21, 20, 10, 0.3, 17), # Between bid and last - ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid - ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid - ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid - ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid - ('bid', 6, 5, None, 1, 5), # last not available - uses bid - ('bid', 6, 5, None, 0, 5), # last not available - uses bid -]) -def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, - last, last_ab, expected) -> None: - caplog.set_level(logging.DEBUG) - default_conf['bid_strategy']['ask_last_balance'] = last_ab - default_conf['bid_strategy']['price_side'] = side - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': ask, 'last': last, 'bid': bid}) - - assert freqtrade.get_buy_rate('ETH/BTC', True) == expected - assert not log_has("Using cached buy rate for ETH/BTC.", caplog) - - assert freqtrade.get_buy_rate('ETH/BTC', False) == expected - assert log_has("Using cached buy rate for ETH/BTC.", caplog) - # Running a 2nd time with Refresh on! - caplog.clear() - assert freqtrade.get_buy_rate('ETH/BTC', True) == expected - assert not log_has("Using cached buy rate for ETH/BTC.", caplog) - - def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -803,13 +760,10 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order stake_amount = 2 bid = 0.11 buy_rate_mock = MagicMock(return_value=bid) - mocker.patch.multiple( - 'freqtrade.freqtradebot.FreqtradeBot', - get_buy_rate=buy_rate_mock, - ) buy_mm = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', + get_buy_rate=buy_rate_mock, fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, @@ -900,7 +854,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order assert not freqtrade.execute_buy(pair, stake_amount) # Fail to get price... - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_buy_rate', MagicMock(return_value=0.0)) + mocker.patch('freqtrade.exchange.Exchange.get_buy_rate', MagicMock(return_value=0.0)) with pytest.raises(PricingError, match="Could not determine buy price."): freqtrade.execute_buy(pair, stake_amount) @@ -908,10 +862,6 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.freqtradebot.FreqtradeBot', - get_buy_rate=MagicMock(return_value=0.11), - ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ @@ -920,6 +870,7 @@ def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) - 'last': 0.00001172 }), buy=MagicMock(return_value=limit_buy_order), + get_buy_rate=MagicMock(return_value=0.11), get_min_pair_stake_amount=MagicMock(return_value=1), get_fee=fee, ) @@ -2523,7 +2474,7 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: 'freqtrade.exchange.Exchange', cancel_order=cancel_order_mock, ) - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', return_value=0.245441) + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', return_value=0.245441) freqtrade = FreqtradeBot(default_conf) @@ -2583,6 +2534,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2646,6 +2598,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2698,6 +2651,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2800,6 +2754,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke price_to_precision=lambda s, x, y: y, stoploss=stoploss, cancel_stoploss_order=cancel_order, + _is_dry_limit_order_filled=MagicMock(side_effect=[True, False]), ) freqtrade = FreqtradeBot(default_conf) @@ -2842,6 +2797,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f get_fee=fee, amount_to_precision=lambda s, x, y: y, price_to_precision=lambda s, x, y: y, + _is_dry_limit_order_filled=MagicMock(side_effect=[False, True]), ) stoploss = MagicMock(return_value={ @@ -2910,6 +2866,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -3515,6 +3472,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_b }), buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) default_conf['ask_strategy'] = { 'ignore_roi_if_buy_signal': False @@ -3978,7 +3936,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_buy_rate('ETH/BTC', True) == 0.043935 + assert freqtrade.exchange.get_buy_rate('ETH/BTC', True) == 0.043935 assert ticker_mock.call_count == 0 @@ -4000,7 +3958,7 @@ def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None freqtrade = FreqtradeBot(default_conf) # orderbook shall be used even if tickers would be lower. with pytest.raises(PricingError): - freqtrade.get_buy_rate('ETH/BTC', refresh=True) + freqtrade.exchange.get_buy_rate('ETH/BTC', refresh=True) assert log_has_re(r'Buy Price from orderbook could not be determined.', caplog) @@ -4072,108 +4030,6 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_o assert log_has('Sell Price at location 1 from orderbook could not be determined.', caplog) -@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', [ - ('bid', 12.0, 11.0, 11.5, 0.0, 11.0), # full bid side - ('bid', 12.0, 11.0, 11.5, 1.0, 11.5), # full last side - ('bid', 12.0, 11.0, 11.5, 0.5, 11.25), # between bid and lat - ('bid', 12.0, 11.2, 10.5, 0.0, 11.2), # Last smaller than bid - ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid - ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid - ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), - ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side - ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side - ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat - ('ask', 12.2, 11.2, 10.5, 0.0, 12.2), # Last smaller than ask - ('ask', 12.0, 11.0, 10.5, 1.0, 12.0), # Last smaller than ask - uses ask - ('ask', 12.0, 11.2, 10.5, 0.5, 12.0), # Last smaller than ask - uses ask - ('ask', 10.0, 11.0, 11.0, 0.0, 10.0), - ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), - ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), - ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), -]) -def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, - last, last_ab, expected) -> None: - caplog.set_level(logging.DEBUG) - - default_conf['ask_strategy']['price_side'] = side - default_conf['ask_strategy']['bid_last_balance'] = last_ab - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': ask, 'bid': bid, 'last': last}) - pair = "ETH/BTC" - - # Test regular mode - ft = get_patched_freqtradebot(mocker, default_conf) - rate = ft.get_sell_rate(pair, True) - assert not log_has("Using cached sell rate for ETH/BTC.", caplog) - assert isinstance(rate, float) - assert rate == expected - # Use caching - rate = ft.get_sell_rate(pair, False) - assert rate == expected - assert log_has("Using cached sell rate for ETH/BTC.", caplog) - - -@pytest.mark.parametrize('side,expected', [ - ('bid', 0.043936), # Value from order_book_l2 fiture - bids side - ('ask', 0.043949), # Value from order_book_l2 fiture - asks side -]) -def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2): - caplog.set_level(logging.DEBUG) - # Test orderbook mode - default_conf['ask_strategy']['price_side'] = side - default_conf['ask_strategy']['use_order_book'] = True - default_conf['ask_strategy']['order_book_min'] = 1 - default_conf['ask_strategy']['order_book_max'] = 2 - pair = "ETH/BTC" - mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) - ft = get_patched_freqtradebot(mocker, default_conf) - rate = ft.get_sell_rate(pair, True) - assert not log_has("Using cached sell rate for ETH/BTC.", caplog) - assert isinstance(rate, float) - assert rate == expected - rate = ft.get_sell_rate(pair, False) - assert rate == expected - assert log_has("Using cached sell rate for ETH/BTC.", caplog) - - -def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog): - # Test orderbook mode - default_conf['ask_strategy']['price_side'] = 'ask' - default_conf['ask_strategy']['use_order_book'] = True - default_conf['ask_strategy']['order_book_min'] = 1 - default_conf['ask_strategy']['order_book_max'] = 2 - pair = "ETH/BTC" - # Test What happens if the exchange returns an empty orderbook. - mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', - return_value={'bids': [[]], 'asks': [[]]}) - ft = get_patched_freqtradebot(mocker, default_conf) - with pytest.raises(PricingError): - ft.get_sell_rate(pair, True) - assert log_has("Sell Price at location from orderbook could not be determined.", caplog) - - -def test_get_sell_rate_exception(default_conf, mocker, caplog): - # Ticker on one side can be empty in certain circumstances. - default_conf['ask_strategy']['price_side'] = 'ask' - pair = "ETH/BTC" - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': None, 'bid': 0.12, 'last': None}) - ft = get_patched_freqtradebot(mocker, default_conf) - with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): - ft.get_sell_rate(pair, True) - - ft.config['ask_strategy']['price_side'] = 'bid' - assert ft.get_sell_rate(pair, True) == 0.12 - # Reverse sides - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': 0.13, 'bid': None, 'last': None}) - with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): - ft.get_sell_rate(pair, True) - - ft.config['ask_strategy']['price_side'] = 'ask' - assert ft.get_sell_rate(pair, True) == 0.13 - - def test_startup_state(default_conf, mocker): default_conf['pairlist'] = {'method': 'VolumePairList', 'config': {'number_assets': 20} diff --git a/tests/test_integration.py b/tests/test_integration.py index 33b3e1140..b12959a03 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -2,9 +2,10 @@ from unittest.mock import MagicMock import pytest +from freqtrade.enums import SellType from freqtrade.persistence import Trade from freqtrade.rpc.rpc import RPC -from freqtrade.strategy.interface import SellCheckTuple, SellType +from freqtrade.strategy.interface import SellCheckTuple from tests.conftest import get_patched_freqtradebot, patch_get_signal diff --git a/tests/test_main.py b/tests/test_main.py index d52dcaf79..3546a3bab 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,10 +7,10 @@ from unittest.mock import MagicMock, PropertyMock import pytest from freqtrade.commands import Arguments +from freqtrade.enums import State from freqtrade.exceptions import FreqtradeException, OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.main import main -from freqtrade.state import State from freqtrade.worker import Worker from tests.conftest import (log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 669f220bb..1576aaa5a 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock import arrow import pytest -from sqlalchemy import create_engine, inspect +from sqlalchemy import create_engine, inspect, text from freqtrade import constants from freqtrade.exceptions import DependencyException, OperationalException @@ -486,9 +486,10 @@ def test_migrate_old(mocker, default_conf, fee): mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format - engine.execute(create_table_old) - engine.execute(insert_table_old) - engine.execute(insert_table_old2) + with engine.begin() as connection: + connection.execute(text(create_table_old)) + connection.execute(text(insert_table_old)) + connection.execute(text(insert_table_old2)) # Run init to test migration init_db(default_conf['db_url'], default_conf['dry_run']) @@ -579,15 +580,16 @@ def test_migrate_new(mocker, default_conf, fee, caplog): mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format - engine.execute(create_table_old) - engine.execute("create index ix_trades_is_open on trades(is_open)") - engine.execute("create index ix_trades_pair on trades(pair)") - engine.execute(insert_table_old) + with engine.begin() as connection: + connection.execute(text(create_table_old)) + connection.execute(text("create index ix_trades_is_open on trades(is_open)")) + connection.execute(text("create index ix_trades_pair on trades(pair)")) + connection.execute(text(insert_table_old)) - # fake previous backup - engine.execute("create table trades_bak as select * from trades") + # fake previous backup + connection.execute(text("create table trades_bak as select * from trades")) - engine.execute("create table trades_bak1 as select * from trades") + connection.execute(text("create table trades_bak1 as select * from trades")) # Run init to test migration init_db(default_conf['db_url'], default_conf['dry_run']) @@ -629,47 +631,49 @@ def test_migrate_new(mocker, default_conf, fee, caplog): caplog.clear() # Drop latest column - engine.execute("alter table orders rename to orders_bak") + with engine.begin() as connection: + connection.execute(text("alter table orders rename to orders_bak")) inspector = inspect(engine) - for index in inspector.get_indexes('orders_bak'): - engine.execute(f"drop index {index['name']}") - # Recreate table - engine.execute(""" - CREATE TABLE orders ( - id INTEGER NOT NULL, - ft_trade_id INTEGER, - ft_order_side VARCHAR NOT NULL, - ft_pair VARCHAR NOT NULL, - ft_is_open BOOLEAN NOT NULL, - order_id VARCHAR NOT NULL, - status VARCHAR, - symbol VARCHAR, - order_type VARCHAR, - side VARCHAR, - price FLOAT, - amount FLOAT, - filled FLOAT, - remaining FLOAT, - cost FLOAT, - order_date DATETIME, - order_filled_date DATETIME, - order_update_date DATETIME, - PRIMARY KEY (id), - CONSTRAINT _order_pair_order_id UNIQUE (ft_pair, order_id), - FOREIGN KEY(ft_trade_id) REFERENCES trades (id) - ) - """) + with engine.begin() as connection: + for index in inspector.get_indexes('orders_bak'): + connection.execute(text(f"drop index {index['name']}")) + # Recreate table + connection.execute(text(""" + CREATE TABLE orders ( + id INTEGER NOT NULL, + ft_trade_id INTEGER, + ft_order_side VARCHAR NOT NULL, + ft_pair VARCHAR NOT NULL, + ft_is_open BOOLEAN NOT NULL, + order_id VARCHAR NOT NULL, + status VARCHAR, + symbol VARCHAR, + order_type VARCHAR, + side VARCHAR, + price FLOAT, + amount FLOAT, + filled FLOAT, + remaining FLOAT, + cost FLOAT, + order_date DATETIME, + order_filled_date DATETIME, + order_update_date DATETIME, + PRIMARY KEY (id), + CONSTRAINT _order_pair_order_id UNIQUE (ft_pair, order_id), + FOREIGN KEY(ft_trade_id) REFERENCES trades (id) + ) + """)) - engine.execute(""" - insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, - symbol, order_type, side, price, amount, filled, remaining, cost, order_date, - order_filled_date, order_update_date) - select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, - symbol, order_type, side, price, amount, filled, remaining, cost, order_date, - order_filled_date, order_update_date - from orders_bak - """) + connection.execute(text(""" + insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, + symbol, order_type, side, price, amount, filled, remaining, cost, order_date, + order_filled_date, order_update_date) + select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, + symbol, order_type, side, price, amount, filled, remaining, cost, order_date, + order_filled_date, order_update_date + from orders_bak + """)) # Run init to test migration init_db(default_conf['db_url'], default_conf['dry_run']) @@ -722,8 +726,9 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format - engine.execute(create_table_old) - engine.execute(insert_table_old) + with engine.begin() as connection: + connection.execute(text(create_table_old)) + connection.execute(text(insert_table_old)) # Run init to test migration init_db(default_conf['db_url'], default_conf['dry_run']) @@ -1288,6 +1293,7 @@ def test_Trade_object_idem(): excludes = ( 'delete', 'session', + 'commit', 'query', 'open_date', 'get_best_pair', diff --git a/tests/test_plotting.py b/tests/test_plotting.py index a22c8c681..20f159e3a 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -460,7 +460,11 @@ def test_plot_profit(default_conf, mocker, testdatadir): assert store_mock.call_count == 1 assert profit_mock.call_args_list[0][0][0] == default_conf['pairs'] - assert store_mock.call_args_list[0][1]['auto_open'] is True + assert store_mock.call_args_list[0][1]['auto_open'] is False + + del default_conf['timeframe'] + with pytest.raises(OperationalException, match=r"Timeframe must be set.*--timeframe.*"): + plot_profit(default_conf) @pytest.mark.parametrize("ind1,ind2,plot_conf,exp", [ diff --git a/tests/test_worker.py b/tests/test_worker.py index 839f7cdac..c3773d296 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -3,7 +3,7 @@ import time from unittest.mock import MagicMock, PropertyMock from freqtrade.data.dataprovider import DataProvider -from freqtrade.state import State +from freqtrade.enums import State from freqtrade.worker import Worker from tests.conftest import get_patched_worker, log_has, log_has_re