diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..b333dc19d --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,18 @@ +FROM freqtradeorg/freqtrade:develop + +# Install dependencies +COPY requirements-dev.txt /freqtrade/ +RUN apt-get update \ + && apt-get -y install git sudo vim \ + && apt-get clean \ + && pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \ + && useradd -u 1000 -U -m ftuser \ + && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \ + && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \ + && echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \ + && chown ftuser: -R /home/ftuser/ + +USER ftuser + +# Empty the ENTRYPOINT to allow all commands +ENTRYPOINT [] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..1882e3bdf --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,44 @@ +{ + "name": "freqtrade Develop", + + "dockerComposeFile": [ + "docker-compose.yml" + ], + + "service": "ft_vscode", + + "workspaceFolder": "/freqtrade/", + + "settings": { + "terminal.integrated.shell.linux": "/bin/bash", + "editor.insertSpaces": true, + "files.trimTrailingWhitespace": true, + "[markdown]": { + "files.trimTrailingWhitespace": false, + }, + "python.pythonPath": "/usr/local/bin/python", + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "davidanson.vscode-markdownlint", + "ms-azuretools.vscode-docker", + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Uncomment the next line if you want start specific services in your Docker Compose config. + // "runServices": [], + + // Uncomment the next line if you want to keep your containers running after VS Code shuts down. + // "shutdownAction": "none", + + // Uncomment the next line to run commands after the container is created - for example installing curl. + // "postCreateCommand": "sudo apt-get update && apt-get install -y git", + + // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "ftuser" +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..7b5e64609 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,24 @@ +--- +version: '3' +services: + ft_vscode: + build: + context: .. + dockerfile: ".devcontainer/Dockerfile" + volumes: + # Allow git usage within container + - "/home/${USER}/.ssh:/home/ftuser/.ssh:ro" + - "/home/${USER}/.gitconfig:/home/ftuser/.gitconfig:ro" + - ..:/freqtrade:cached + # Persist bash-history + - freqtrade-vscode-server:/home/ftuser/.vscode-server + - freqtrade-bashhistory:/home/ftuser/commandhistory + # Expose API port + ports: + - "127.0.0.1:8080:8080" + command: /bin/sh -c "while sleep 1000; do :; done" + + +volumes: + freqtrade-vscode-server: + freqtrade-bashhistory: diff --git a/.dockerignore b/.dockerignore index 223b3b110..09f4c9f0c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,4 @@ CONTRIBUTING.md MANIFEST.in README.md freqtrade.service +user_data diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c6141344..392641677 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,11 +4,11 @@ on: push: branches: - master + - stable - develop - - github_actions_tests tags: - release: - types: [published] + release: + types: [published] pull_request: schedule: - cron: '0 5 * * 4' @@ -194,7 +194,7 @@ jobs: steps: - name: Cleanup previous runs on this branch uses: rokroskar/workflow-run-cleanup-action@v0.2.2 - if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.repository == 'freqtrade/freqtrade'" + if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -226,7 +226,7 @@ jobs: - name: Extract branch name shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})" id: extract_branch - name: Build distribution @@ -236,7 +236,7 @@ jobs: - name: Publish to PyPI (Test) uses: pypa/gh-action-pypi-publish@master - if: (steps.extract_branch.outputs.branch == 'master' || github.event_name == 'release') + if: (steps.extract_branch.outputs.branch == 'stable' || github.event_name == 'release') with: user: __token__ password: ${{ secrets.pypi_test_password }} @@ -244,7 +244,7 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@master - if: (steps.extract_branch.outputs.branch == 'master' || github.event_name == 'release') + if: (steps.extract_branch.outputs.branch == 'stable' || github.event_name == 'release') with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/docker_update_readme.yml b/.github/workflows/docker_update_readme.yml index 57a7e591e..95e69be2a 100644 --- a/.github/workflows/docker_update_readme.yml +++ b/.github/workflows/docker_update_readme.yml @@ -2,7 +2,7 @@ name: Update Docker Hub Description on: push: branches: - - master + - stable jobs: dockerHubDescription: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 90594866a..97f62154d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,8 +8,9 @@ Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/ Few pointers for contributions: -- Create your PR against the `develop` branch, not `master`. -- New features need to contain unit tests and must be PEP8 conformant (max-line-length = 100). +- Create your PR against the `develop` branch, not `stable`. +- New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR. +- PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished). If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. @@ -18,7 +19,7 @@ or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. Best start by reading the [documentation](https://www.freqtrade.io/) to get a feel for what is possible with the bot, or head straight to the [Developer-documentation](https://www.freqtrade.io/en/latest/developer/) (WIP) which should help you getting started. -## Before sending the PR: +## Before sending the PR ### 1. Run unit tests @@ -114,6 +115,6 @@ Contributors may be given commit privileges. Preference will be given to those w 1. Access to resources for cross-platform development and testing. 1. Time to devote to the project regularly. -Being a Committer does not grant write permission on `develop` or `master` for security reasons (Users trust Freqtrade with their Exchange API keys). +Being a Committer does not grant write permission on `develop` or `stable` for security reasons (Users trust Freqtrade with their Exchange API keys). After being Committer for some time, a Committer may be named Core Committer and given full repository access. diff --git a/Dockerfile b/Dockerfile index e1220e3b8..2be65274e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.5-slim-buster +FROM python:3.8.6-slim-buster RUN apt-get update \ && apt-get -y install curl build-essential libssl-dev sqlite3 \ @@ -16,13 +16,14 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies -COPY requirements.txt requirements-common.txt requirements-hyperopt.txt /freqtrade/ +COPY requirements.txt requirements-hyperopt.txt /freqtrade/ RUN pip install numpy --no-cache-dir \ && pip install -r requirements-hyperopt.txt --no-cache-dir # Install and execute COPY . /freqtrade/ -RUN pip install -e . --no-cache-dir +RUN pip install -e . --no-cache-dir \ + && mkdir /freqtrade/user_data/ ENTRYPOINT ["freqtrade"] # Default to trade mode CMD [ "trade" ] diff --git a/Dockerfile.armhf b/Dockerfile.armhf index 5c5e4a885..0633008ea 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -17,7 +17,7 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies -COPY requirements.txt requirements-common.txt /freqtrade/ +COPY requirements.txt /freqtrade/ RUN pip install numpy --no-cache-dir \ && pip install -r requirements.txt --no-cache-dir diff --git a/README.md b/README.md index 90f303c6d..feea47299 100644 --- a/README.md +++ b/README.md @@ -127,8 +127,8 @@ Telegram is not mandatory. However, this is a great way to control your bot. Mor The project is currently setup in two main branches: -- `develop` - This branch has often new features, but might also cause breaking changes. -- `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested. +- `develop` - This branch has often new features, but might also contain breaking changes. We try hard to keep this branch as stable as possible. +- `stable` - This branch contains the latest stable release. This branch is generally well tested. - `feat/*` - These are feature branches, which are being worked on heavily. Please don't use these unless you want to test a specific feature. ## Support @@ -171,11 +171,11 @@ Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/ **Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. -**Important:** Always create your PR against the `develop` branch, not `master`. +**Important:** Always create your PR against the `develop` branch, not `stable`. ## Requirements -### Uptodate clock +### Up-to-date clock The clock must be accurate, syncronized to a NTP server very frequently to avoid problems with communication to the exchanges. diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 03a95161b..53e18063c 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -2,6 +2,7 @@ # Replace / with _ to create a valid tag TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") +TAG_PLOT=${TAG}_plot echo "Running for ${TAG}" # Add commit and commit_message to docker container @@ -16,6 +17,7 @@ else docker pull ${IMAGE_NAME}:${TAG} docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . fi +docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot . if [ $? -ne 0 ]; then echo "failed building image" @@ -32,6 +34,7 @@ fi # Tag image for upload docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG +docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT if [ $? -ne 0 ]; then echo "failed tagging image" return 1 diff --git a/config.json.example b/config.json.example index 77a147d0c..ab517b77c 100644 --- a/config.json.example +++ b/config.json.example @@ -7,7 +7,6 @@ "timeframe": "5m", "dry_run": false, "cancel_open_orders_on_exit": false, - "trailing_stop": false, "unfilledtimeout": { "buy": 10, "sell": 30 diff --git a/config_binance.json.example b/config_binance.json.example index 82943749d..f3f8eb659 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -7,7 +7,6 @@ "timeframe": "5m", "dry_run": true, "cancel_open_orders_on_exit": false, - "trailing_stop": false, "unfilledtimeout": { "buy": 10, "sell": 30 diff --git a/config_full.json.example b/config_full.json.example index d5bfd3fe1..659580fb1 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -116,7 +116,16 @@ "telegram": { "enabled": true, "token": "your_telegram_token", - "chat_id": "your_telegram_chat_id" + "chat_id": "your_telegram_chat_id", + "notification_settings": { + "status": "on", + "warning": "on", + "startup": "on", + "buy": "on", + "sell": "on", + "buy_cancel": "on", + "sell_cancel": "on" + } }, "api_server": { "enabled": false, diff --git a/config_kraken.json.example b/config_kraken.json.example index fb983a4a3..fd0b2b95d 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -7,7 +7,6 @@ "timeframe": "5m", "dry_run": true, "cancel_open_orders_on_exit": false, - "trailing_stop": false, "unfilledtimeout": { "buy": 10, "sell": 30 diff --git a/docker-compose.develop.yml b/docker-compose.develop.yml deleted file mode 100644 index 562b5960a..000000000 --- a/docker-compose.develop.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -version: '3' -services: - freqtrade_develop: - build: - context: . - dockerfile: "./Dockerfile.develop" - volumes: - - ".:/freqtrade" - entrypoint: - - "freqtrade" - - freqtrade_bash: - build: - context: . - dockerfile: "./Dockerfile.develop" - volumes: - - ".:/freqtrade" - entrypoint: - - "/bin/bash" diff --git a/docker-compose.yml b/docker-compose.yml index 49d83aa5e..a99aac3c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,10 @@ version: '3' services: freqtrade: - image: freqtradeorg/freqtrade:master + image: freqtradeorg/freqtrade:stable # image: freqtradeorg/freqtrade:develop + # Use plotting image + # image: freqtradeorg/freqtrade:develop_plot # Build step - only needed when additional dependencies are needed # build: # context: . diff --git a/Dockerfile.develop b/docker/Dockerfile.develop similarity index 99% rename from Dockerfile.develop rename to docker/Dockerfile.develop index 8f6871c55..cb49984e2 100644 --- a/Dockerfile.develop +++ b/docker/Dockerfile.develop @@ -2,6 +2,7 @@ FROM freqtradeorg/freqtrade:develop # Install dependencies COPY requirements-dev.txt /freqtrade/ + RUN pip install numpy --no-cache-dir \ && pip install -r requirements-dev.txt --no-cache-dir diff --git a/docker/Dockerfile.jupyter b/docker/Dockerfile.jupyter new file mode 100644 index 000000000..b7499eeef --- /dev/null +++ b/docker/Dockerfile.jupyter @@ -0,0 +1,7 @@ +FROM freqtradeorg/freqtrade:develop_plot + + +RUN pip install jupyterlab --no-cache-dir + +# Empty the ENTRYPOINT to allow all commands +ENTRYPOINT [] diff --git a/docker/Dockerfile.plot b/docker/Dockerfile.plot new file mode 100644 index 000000000..1843efdcb --- /dev/null +++ b/docker/Dockerfile.plot @@ -0,0 +1,10 @@ +ARG sourceimage=develop +FROM freqtradeorg/freqtrade:${sourceimage} + +# Install dependencies +COPY requirements-plot.txt /freqtrade/ + +RUN pip install -r requirements-plot.txt --no-cache-dir + +# Empty the ENTRYPOINT to allow all commands +ENTRYPOINT [] diff --git a/Dockerfile.technical b/docker/Dockerfile.technical similarity index 100% rename from Dockerfile.technical rename to docker/Dockerfile.technical diff --git a/docker/docker-compose-jupyter.yml b/docker/docker-compose-jupyter.yml new file mode 100644 index 000000000..11a01705c --- /dev/null +++ b/docker/docker-compose-jupyter.yml @@ -0,0 +1,16 @@ +--- +version: '3' +services: + ft_jupyterlab: + build: + context: .. + dockerfile: docker/Dockerfile.jupyter + restart: unless-stopped + container_name: freqtrade + ports: + - "127.0.0.1:8888:8888" + volumes: + - "./user_data:/freqtrade/user_data" + # Default command used when running `docker compose up` + command: > + jupyter lab --port=8888 --ip 0.0.0.0 --allow-root diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 40ff3d82b..4a4496bbc 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -5,6 +5,9 @@ This page explains the different parameters of the bot and how to run it. !!! Note If you've used `setup.sh`, don't forget to activate your virtual environment (`source .env/bin/activate`) before running freqtrade commands. +!!! Warning "Up-to-date clock" + The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges. + ## Bot commands ``` diff --git a/docs/configuration.md b/docs/configuration.md index bf141f8e8..d6e26f80e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -375,7 +375,7 @@ Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports exchange markets and trading APIs. The complete up-to-date list can be found in the [CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested by the development team with only Bittrex, Binance and Kraken, - so the these are the only officially supported exhanges: + so the these are the only officially supported exchanges: - [Bittrex](https://bittrex.com/): "bittrex" - [Binance](https://www.binance.com/): "binance" diff --git a/docs/data-analysis.md b/docs/data-analysis.md index fc4693b17..17da98935 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -1,12 +1,22 @@ # Analyzing bot data with Jupyter notebooks -You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/`. +You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/` after initializing the user directory with `freqtrade create-userdir --userdir user_data`. -## Pro tips +## Quick start with docker + +Freqtrade provides a docker-compose file which starts up a jupyter lab server. +You can run this server using the following command: `docker-compose -f docker/docker-compose-jupyter.yml up` + +This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`. +Please use the link that's printed in the console after startup for simplified login. + +For more information, Please visit the [Data analysis with Docker](docker_quickstart.md#data-analayis-using-docker-compose) section. + +### Pro tips * See [jupyter.org](https://jupyter.org/documentation) for usage instructions. * Don't forget to start a Jupyter notebook server from within your conda or venv environment or use [nb_conda_kernels](https://github.com/Anaconda-Platform/nb_conda_kernels)* -* Copy the example notebook before use so your changes don't get clobbered with the next freqtrade update. +* Copy the example notebook before use so your changes don't get overwritten with the next freqtrade update. ### Using virtual environment with system-wide Jupyter installation @@ -28,10 +38,8 @@ ipython kernel install --user --name=freqtrade !!! Note This section is provided for completeness, the Freqtrade Team won't provide full support for problems with this setup and will recommend to install Jupyter in the virtual environment directly, as that is the easiest way to get jupyter notebooks up and running. For help with this setup please refer to the [Project Jupyter](https://jupyter.org/) [documentation](https://jupyter.org/documentation) or [help channels](https://jupyter.org/community). - -## Fine print - -Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually. +!!! Warning + Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually. ## Recommended workflow diff --git a/docs/data-download.md b/docs/data-download.md index a2bbec837..e9c5c1865 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -8,82 +8,121 @@ If no additional parameter is specified, freqtrade will download data for `"1m"` Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. +You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101`). For incremental downloads, the relative approach should be used. + !!! Tip "Tip: Updating existing data" If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. - Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. + Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. ### Usage ``` -usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] - [--pairs-file FILE] [--days INT] [--dl-trades] [--exchange EXCHANGE] - [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]] - [--erase] [--data-format-ohlcv {json,jsongz}] [--data-format-trades {json,jsongz}] - -optional arguments: - -h, --help show this help message and exit - -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] - Show profits for only these pairs. Pairs are space-separated. - --pairs-file FILE File containing a list of pairs to download. - --days INT Download data for given number of days. - --dl-trades Download trades instead of OHLCV data. The bot will resample trades to the desired timeframe as specified as - --timeframes/-t. - --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. - -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...] - Specify which tickers to download. Space-separated list. Default: `1m 5m`. - --erase Clean all existing data for the selected exchange/pairs/timeframes. - --data-format-ohlcv {json,jsongz} - Storage format for downloaded candle (OHLCV) data. (default: `json`). - --data-format-trades {json,jsongz} - Storage format for downloaded trades data. (default: `jsongz`). - -Common arguments: - -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. - -V, --version show program's version number and exit - -c PATH, --config PATH - Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` - to read config from stdin. - -d PATH, --datadir PATH - Path to directory with historical backtesting data. - --userdir PATH, --user-data-dir PATH - Path to userdata directory. -``` - -### Data format - -Freqtrade currently supports 2 dataformats, `json` (plain "text" json files) and `jsongz` (a gzipped version of json files). -By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. - -This can be changed via the `--data-format-ohlcv` and `--data-format-trades` parameters respectivly. - -If the default dataformat has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well. - -!!! Note - You can convert between data-formats using the [convert-data](#subcommand-convert-data) and [convert-trade-data](#subcommand-convert-trade-data) methods. - -#### Subcommand convert data - -``` -usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] - [-p PAIRS [PAIRS ...]] --format-from - {json,jsongz} --format-to {json,jsongz} - [--erase] - [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]] +usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [-p PAIRS [PAIRS ...]] [--pairs-file FILE] + [--days INT] [--timerange TIMERANGE] + [--dl-trades] [--exchange EXCHANGE] + [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] + [--erase] + [--data-format-ohlcv {json,jsongz,hdf5}] + [--data-format-trades {json,jsongz,hdf5}] optional arguments: -h, --help show this help message and exit -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] Show profits for only these pairs. Pairs are space- separated. - --format-from {json,jsongz} + --pairs-file FILE File containing a list of pairs to download. + --days INT Download data for given number of days. + --timerange TIMERANGE + Specify what timerange of data to use. + --dl-trades Download trades instead of OHLCV data. The bot will + resample trades to the desired timeframe as specified + as --timeframes/-t. + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no + config is provided. + -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...] + Specify which tickers to download. Space-separated + list. Default: `1m 5m`. + --erase Clean all existing data for the selected + exchange/pairs/timeframes. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `json`). + --data-format-trades {json,jsongz,hdf5} + Storage format for downloaded trades data. (default: + `jsongz`). + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` + +!!! Note "Startup period" + `download-data` is a strategy-independent command. The idea is to download a big chunk of data once, and then iteratively increase the amount of data stored. + + For that reason, `download-data` does not care about the "startup-period" defined in a strategy. It's up to the user to download additional days if the backtest should start at a specific point in time (while respecting startup period). + +### Data format + +Freqtrade currently supports 3 data-formats for both OHLCV and trades data: + +* `json` (plain "text" json files) +* `jsongz` (a gzip-zipped version of json files) +* `hdf5` (a high performance datastore) + +By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. + +This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively. +To persist this change, you can should also add the following snippet to your configuration, so you don't have to insert the above arguments each time: + +``` jsonc + // ... + "dataformat_ohlcv": "hdf5", + "dataformat_trades": "hdf5", + // ... +``` + +If the default data-format has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well. + +!!! Note + You can convert between data-formats using the [convert-data](#sub-command-convert-data) and [convert-trade-data](#sub-command-convert-trade-data) methods. + +#### Sub-command convert data + +``` +usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [-p PAIRS [PAIRS ...]] --format-from + {json,jsongz,hdf5} --format-to + {json,jsongz,hdf5} [--erase] + [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space- + separated. + --format-from {json,jsongz,hdf5} Source format for data conversion. - --format-to {json,jsongz} + --format-to {json,jsongz,hdf5} Destination format for data conversion. --erase Clean all existing data for the selected exchange/pairs/timeframes. - -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...] + -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...] Specify which tickers to download. Space-separated list. Default: `1m 5m`. @@ -94,9 +133,10 @@ Common arguments: details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -112,23 +152,23 @@ It'll also remove original json data files (`--erase` parameter). freqtrade convert-data --format-from json --format-to jsongz --datadir ~/.freqtrade/data/binance -t 5m 15m --erase ``` -#### Subcommand convert-trade data +#### Sub-command convert trade data ``` usage: freqtrade convert-trade-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] --format-from - {json,jsongz} --format-to {json,jsongz} - [--erase] + {json,jsongz,hdf5} --format-to + {json,jsongz,hdf5} [--erase] optional arguments: -h, --help show this help message and exit -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] Show profits for only these pairs. Pairs are space- separated. - --format-from {json,jsongz} + --format-from {json,jsongz,hdf5} Source format for data conversion. - --format-to {json,jsongz} + --format-to {json,jsongz,hdf5} Destination format for data conversion. --erase Clean all existing data for the selected exchange/pairs/timeframes. @@ -140,13 +180,15 @@ Common arguments: details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH Path to userdata directory. + ``` ##### Example converting trades @@ -158,21 +200,21 @@ It'll also remove original jsongz data files (`--erase` parameter). freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase ``` -### Subcommand list-data +### Sub-command list-data -You can get a list of downloaded data using the `list-data` subcommand. +You can get a list of downloaded data using the `list-data` sub-command. ``` usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] - [--data-format-ohlcv {json,jsongz}] + [--data-format-ohlcv {json,jsongz,hdf5}] [-p PAIRS [PAIRS ...]] optional arguments: -h, --help show this help message and exit --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. - --data-format-ohlcv {json,jsongz} + --data-format-ohlcv {json,jsongz,hdf5} Storage format for downloaded candle (OHLCV) data. (default: `json`). -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] @@ -194,6 +236,7 @@ Common arguments: Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH Path to userdata directory. + ``` #### Example list-data @@ -249,15 +292,16 @@ This will download historical candle (OHLCV) data for all the currency pairs you ### Other Notes - To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. -- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust ratelimits etc.) +- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.) - To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. - To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days). +- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. Eventually set end dates are ignored. - Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. ### Trades (tick) data -By default, `download-data` subcommand downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API. +By default, `download-data` sub-command downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API. This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes. Since this data is large by default, the files use gzip by default. They are stored in your data-directory with the naming convention of `-trades.json.gz` (`ETH_BTC-trades.json.gz`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository. diff --git a/docs/deprecated.md b/docs/deprecated.md index 44f0b686a..312f2c74f 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -32,4 +32,4 @@ The old section of configuration parameters (`"pairlist"`) has been deprecated i ### deprecation of bidVolume and askVolume from volume-pairlist -Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4. +Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4, and have been removed in 2020.9. diff --git a/docs/developer.md b/docs/developer.md index f09ae2c76..788e961cd 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -10,13 +10,35 @@ Documentation is available at [https://freqtrade.io](https://www.freqtrade.io/) Special fields for the documentation (like Note boxes, ...) can be found [here](https://squidfunk.github.io/mkdocs-material/extensions/admonition/). +To test the documentation locally use the following commands. + +``` bash +pip install -r docs/requirements-docs.txt +mkdocs serve +``` + +This will spin up a local server (usually on port 8000) so you can see if everything looks as you'd like it to. + ## Developer setup -To configure a development environment, best use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ". -Alternatively (if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`. +To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ". +Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`. This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`. +### Devcontainer setup + +The fastest and easiest way to get started is to use [VSCode](https://code.visualstudio.com/) with the Remote container extension. +This gives developers the ability to start the bot with all required dependencies *without* needing to install any freqtrade specific dependencies on your local machine. + +#### Devcontainer dependencies + +* [VSCode](https://code.visualstudio.com/) +* [docker](https://docs.docker.com/install/) +* [Remote container extension documentation](https://code.visualstudio.com/docs/remote) + +For more information about the [Remote container extension](https://code.visualstudio.com/docs/remote), best consult the documentation. + ### Tests New code should be covered by basic unittests. Depending on the complexity of the feature, Reviewers may request more in-depth unittests. @@ -41,50 +63,6 @@ def test_method_to_test(caplog): ``` -### Local docker usage - -The fastest and easiest way to start up is to use docker-compose.develop which gives developers the ability to start the bot up with all the required dependencies, *without* needing to install any freqtrade specific dependencies on your local machine. - -#### Install - -* [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) -* [docker](https://docs.docker.com/install/) -* [docker-compose](https://docs.docker.com/compose/install/) - -#### Starting the bot -##### Use the develop dockerfile - -``` bash -rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml -``` - -#### Docker Compose - -##### Starting - -``` bash -docker-compose up -``` - -![Docker compose up](https://user-images.githubusercontent.com/419355/65456322-47f63a80-de06-11e9-90c6-3c74d1bad0b8.png) - -##### Rebuilding - -``` bash -docker-compose build -``` - -##### Execing (effectively SSH into the container) - -The `exec` command requires that the container already be running, if you want to start it -that can be effected by `docker-compose up` or `docker-compose run freqtrade_develop` - -``` bash -docker-compose exec freqtrade_develop /bin/bash -``` - -![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png) - ## ErrorHandling Freqtrade Exceptions all inherit from `FreqtradeException`. @@ -110,6 +88,8 @@ Below is an outline of exception inheritance hierarchy: | +---+ InvalidOrderException | | | +---+ RetryableOrderError +| | +| +---+ InsufficientFundsError | +---+ StrategyError ``` @@ -127,7 +107,7 @@ First of all, have a look at the [VolumePairList](https://github.com/freqtrade/f This is a simple Handler, which however serves as a good example on how to start developing. -Next, modify the classname of the Handler (ideally align this with the module filename). +Next, modify the class-name of the Handler (ideally align this with the module filename). The base-class provides an instance of the exchange (`self._exchange`) the pairlist manager (`self._pairlistmanager`), as well as the main configuration (`self._config`), the pairlist dedicated configuration (`self._pairlistconfig`) and the absolute position within the list of pairlists. @@ -147,7 +127,7 @@ Configuration for the chain of Pairlist Handlers is done in the bot configuratio By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the pairlist. Please follow this to ensure a consistent user experience. -Additional parameters can be configured as needed. For instance, `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic. +Additional parameters can be configured as needed. For instance, `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successful and dynamic. #### short_desc @@ -163,7 +143,7 @@ This is called with each iteration of the bot (only if the Pairlist Handler is a It must return the resulting pairlist (which may then be passed into the chain of Pairlist Handlers). -Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected. +Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the end-result is not shorter than expected. #### filter_pairlist @@ -171,13 +151,13 @@ This method is called for each Pairlist Handler in the chain by the pairlist man This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. -It get's passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`. +It gets passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`. The default implementation in the base class simply calls the `_validate_pair()` method for each pair in the pairlist, but you may override it. So you should either implement the `_validate_pair()` in your Pairlist Handler or override `filter_pairlist()` to do something else. If overridden, it must return the resulting pairlist (which may then be passed into the next Pairlist Handler in the chain). -Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected. +Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the end result is not shorter than expected. In `VolumePairList`, this implements different methods of sorting, does early validation so only the expected number of pairs is returned. @@ -201,7 +181,7 @@ Most exchanges supported by CCXT should work out of the box. Check if the new exchange supports Stoploss on Exchange orders through their API. -Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselfs. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects. +Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselves. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects. ### Incomplete candles @@ -251,13 +231,14 @@ jupyter nbconvert --ClearOutputPreprocessor.enabled=True --to markdown freqtrade This documents some decisions taken for the CI Pipeline. * CI runs on all OS variants, Linux (ubuntu), macOS and Windows. -* Docker images are build for the branches `master` and `develop`. -* Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:master_pi` and `develop_pi`. +* Docker images are build for the branches `stable` and `develop`. +* Docker images containing Plot dependencies are also available as `stable_plot` and `develop_plot`. +* Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:stable_pi` and `develop_pi`. * Docker images contain a file, `/freqtrade/freqtrade_commit` containing the commit this image is based of. * Full docker image rebuilds are run once a week via schedule. * Deployments run on ubuntu. * ta-lib binaries are contained in the build_helpers directory to avoid fails related to external unavailability. -* All tests must pass for a PR to be merged to `master` or `develop`. +* All tests must pass for a PR to be merged to `stable` or `develop`. ## Creating a release @@ -274,21 +255,22 @@ git checkout -b new_release Determine if crucial bugfixes have been made between this commit and the current state, and eventually cherry-pick these. +* Merge the release branch (stable) into this branch. * Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi. * Commit this part -* push that branch to the remote and create a PR against the master branch +* push that branch to the remote and create a PR against the stable branch ### Create changelog from git commits !!! Note - Make sure that the master branch is uptodate! + Make sure that the `stable` branch is up-to-date! ``` bash # Needs to be done before merging / pulling that branch. -git log --oneline --no-decorate --no-merges master..new_release +git log --oneline --no-decorate --no-merges stable..new_release ``` -To keep the release-log short, best wrap the full git changelog into a collapsible details secction. +To keep the release-log short, best wrap the full git changelog into a collapsible details section. ```markdown
@@ -301,17 +283,20 @@ To keep the release-log short, best wrap the full git changelog into a collapsib ### Create github release / tag -Once the PR against master is merged (best right after merging): +Once the PR against stable is merged (best right after merging): * Use the button "Draft a new release" in the Github UI (subsection releases). * Use the version-number specified as tag. -* Use "master" as reference (this step comes after the above PR is merged). +* Use "stable" as reference (this step comes after the above PR is merged). * Use the above changelog as release comment (as codeblock) ## Releases ### pypi +!!! Note + This process is now automated as part of Github Actions. + To create a pypi release, please run the following commands: Additional requirement: `wheel`, `twine` (for uploading), account on pypi with proper permissions. diff --git a/docs/docker.md b/docs/docker.md index 92478088a..f4699cf4c 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -1,145 +1,7 @@ -# Using Freqtrade with Docker - -## Install Docker - -Start by downloading and installing Docker CE for your platform: - -* [Mac](https://docs.docker.com/docker-for-mac/install/) -* [Windows](https://docs.docker.com/docker-for-windows/install/) -* [Linux](https://docs.docker.com/install/) - -Optionally, [docker-compose](https://docs.docker.com/compose/install/) should be installed and available to follow the [docker quick start guide](#docker-quick-start). - -Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. - -## Freqtrade with docker-compose - -Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage. - -!!! Note - The following section assumes that docker and docker-compose is installed and available to the logged in user. - -!!! Note - All below comands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file. - -!!! Note "Docker on Raspberry" - If you're running freqtrade on a Raspberry PI, you must change the image from `freqtradeorg/freqtrade:master` to `freqtradeorg/freqtrade:master_pi` or `freqtradeorg/freqtrade:develop_pi`, otherwise the image will not work. - -### Docker quick start - -Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory. - -``` bash -mkdir ft_userdata -cd ft_userdata/ -# Download the docker-compose file from the repository -curl https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docker-compose.yml -o docker-compose.yml - -# Pull the freqtrade image -docker-compose pull - -# Create user directory structure -docker-compose run --rm freqtrade create-userdir --userdir user_data - -# Create configuration - Requires answering interactive questions -docker-compose run --rm freqtrade new-config --config user_data/config.json -``` - -The above snippet creates a new directory called "ft_userdata", downloads the latest compose file and pulls the freqtrade image. -The last 2 steps in the snippet create the directory with user-data, as well as (interactively) the default configuration based on your selections. - -!!! Note - You can edit the configuration at any time, which is available as `user_data/config.json` (within the directory `ft_userdata`) when using the above configuration. - -#### Adding your strategy - -The configuration is now available as `user_data/config.json`. -You should now copy your strategy to `user_data/strategies/` - and add the Strategy class name to the `docker-compose.yml` file, replacing `SampleStrategy`. If you wish to run the bot with the SampleStrategy, just leave it as it is. - -!!! Warning - The `SampleStrategy` is there for your reference and give you ideas for your own strategy. - Please always backtest the strategy and use dry-run for some time before risking real money! - -Once this is done, you're ready to launch the bot in trading mode (Dry-run or Live-trading, depending on your answer to the corresponding question you made above). - -``` bash -docker-compose up -d -``` - -#### Docker-compose logs - -Logs will be written to `user_data/logs/freqtrade.log`. -Alternatively, you can check the latest logs using `docker-compose logs -f`. - -#### Database - -The database will be in the user_data directory as well, and will be called `user_data/tradesv3.sqlite`. - -#### Updating freqtrade with docker-compose - -To update freqtrade when using docker-compose is as simple as running the following 2 commands: - -``` bash -# Download the latest image -docker-compose pull -# Restart the image -docker-compose up -d -``` - -This will first pull the latest image, and will then restart the container with the just pulled version. - -!!! Note - You should always check the changelog for breaking changes / manual interventions required and make sure the bot starts correctly after the update. - -#### Going from here - -Advanced users may edit the docker-compose file further to include all possible options or arguments. - -All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. - -!!! Note "`docker-compose run --rm`" - Including `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). - -##### Example: Download data with docker-compose - -Download backtesting data for 5 days for the pair ETH/BTC and 1h timeframe from Binance. The data will be stored in the directory `user_data/data/` on the host. - -``` bash -docker-compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h -``` - -Head over to the [Data Downloading Documentation](data-download.md) for more details on downloading data. - -##### Example: Backtest with docker-compose - -Run backtesting in docker-containers for SampleStrategy and specified timerange of historical data, on 5m timeframe: - -``` bash -docker-compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m -``` - -Head over to the [Backtesting Documentation](backtesting.md) to learn more. - -#### Additional dependencies with docker-compose - -If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host. -For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) for an example). - -You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions. - -``` yaml - image: freqtrade_custom - build: - context: . - dockerfile: "./Dockerfile." -``` - -You can then run `docker-compose build` to build the docker image, and run it using the commands described above. - ## Freqtrade with docker without docker-compose !!! Warning - The below documentation is provided for completeness and assumes that you are somewhat familiar with running docker containers. If you're just starting out with docker, we recommend to follow the [Freqtrade with docker-compose](#freqtrade-with-docker-compose) instructions. + The below documentation is provided for completeness and assumes that you are familiar with running docker containers. If you're just starting out with Docker, we recommend to follow the [Quickstart](docker.md) instructions. ### Download the official Freqtrade docker image @@ -148,9 +10,9 @@ Pull the image from docker hub. Branches / tags available can be checked out on [Dockerhub tags page](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). ```bash -docker pull freqtradeorg/freqtrade:develop +docker pull freqtradeorg/freqtrade:stable # Optionally tag the repository so the run-commands remain shorter -docker tag freqtradeorg/freqtrade:develop freqtrade +docker tag freqtradeorg/freqtrade:stable freqtrade ``` To update the image, simply run the above commands again and restart your running container. @@ -158,7 +20,7 @@ To update the image, simply run the above commands again and restart your runnin Should you require additional libraries, please [build the image yourself](#build-your-own-docker-image). !!! Note "Docker image update frequency" - The official docker images with tags `master`, `develop` and `latest` are automatically rebuild once a week to keep the base image uptodate. + The official docker images with tags `stable`, `develop` and `latest` are automatically rebuild once a week to keep the base image up-to-date. In addition to that, every merge to `develop` will trigger a rebuild for `develop` and `latest`. ### Prepare the configuration files @@ -190,39 +52,38 @@ cp -n config.json.example config.json #### Create your database file -Production +=== "Dry-Run" + ``` bash + touch tradesv3.dryrun.sqlite + ``` -```bash -touch tradesv3.sqlite -```` +=== "Production" + ``` bash + touch tradesv3.sqlite + ``` -Dry-Run -```bash -touch tradesv3.dryrun.sqlite -``` - -!!! Note - Make sure to use the path to this file when starting the bot in docker. +!!! Warning "Database File Path" + Make sure to use the path to the correct database file when starting the bot in Docker. ### Build your own Docker image Best start by pulling the official docker image from dockerhub as explained [here](#download-the-official-docker-image) to speed up building. -To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image. +To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image. ```bash -docker build -t freqtrade -f Dockerfile.technical . +docker build -t freqtrade -f docker/Dockerfile.technical . ``` -If you are developing using Docker, use `Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies: +If you are developing using Docker, use `docker/Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies: ```bash -docker build -f Dockerfile.develop -t freqtrade-dev . +docker build -f docker/Dockerfile.develop -t freqtrade-dev . ``` -!!! Note - For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see the "5. Run a restartable docker image" section) to keep it between updates. +!!! Warning "Include your config file manually" + For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see [5. Run a restartable docker image](#run-a-restartable-docker-image)") to keep it between updates. #### Verify the Docker image @@ -243,37 +104,36 @@ docker run --rm -v `pwd`/config.json:/freqtrade/config.json -it freqtrade ``` !!! Warning - In this example, the database will be created inside the docker instance and will be lost when you will refresh your image. + In this example, the database will be created inside the docker instance and will be lost when you refresh your image. #### Adjust timezone By default, the container will use UTC timezone. -Should you find this irritating please add the following to your docker commands: +If you would like to change the timezone use the following commands: -##### Linux +=== "Linux" + ``` bash + -v /etc/timezone:/etc/timezone:ro -``` bash --v /etc/timezone:/etc/timezone:ro + # Complete command: + docker run --rm -v /etc/timezone:/etc/timezone:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade + ``` -# Complete command: -docker run --rm -v /etc/timezone:/etc/timezone:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` +=== "MacOS" + ```bash + docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade + ``` -##### MacOS - -There is known issue in OSX Docker versions after 17.09.1, whereby `/etc/localtime` cannot be shared causing Docker to not start. A work-around for this is to start with the following cmd. - -```bash -docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` - -More information on this docker issue and work-around can be read [here](https://github.com/docker/for-mac/issues/2396). +!!! Note "MacOS Issues" + The OSX Docker versions after 17.09.1 have a known issue whereby `/etc/localtime` cannot be shared causing Docker to not start.
+ A work-around for this is to start with the MacOS command above + More information on this docker issue and work-around can be read [here](https://github.com/docker/for-mac/issues/2396). ### Run a restartable docker image To run a restartable instance in the background (feel free to place your configuration and database files wherever it feels comfortable on your filesystem). -#### Move your config file and database +#### 1. Move your config file and database The following will assume that you place your configuration / database files to `~/.freqtrade`, which is a hidden directory in your home directory. Feel free to use a different directory and replace the directory in the upcomming commands. @@ -283,7 +143,7 @@ mv config.json ~/.freqtrade mv tradesv3.sqlite ~/.freqtrade ``` -#### Run the docker image +#### 2. Run the docker image ```bash docker run -d \ diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md new file mode 100644 index 000000000..48ee34954 --- /dev/null +++ b/docs/docker_quickstart.md @@ -0,0 +1,191 @@ +# Using Freqtrade with Docker + +## Install Docker + +Start by downloading and installing Docker CE for your platform: + +* [Mac](https://docs.docker.com/docker-for-mac/install/) +* [Windows](https://docs.docker.com/docker-for-windows/install/) +* [Linux](https://docs.docker.com/install/) + +Optionally, [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the [docker quick start guide](#docker-quick-start). + +Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. + +## Freqtrade with docker-compose + +Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage. + +!!! Note + - The following section assumes that `docker` and `docker-compose` are installed and available to the logged in user. + - All below commands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file. + +### Docker quick start + +Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory. + +=== "PC/MAC/Linux" + ``` bash + mkdir ft_userdata + cd ft_userdata/ + # Download the docker-compose file from the repository + curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml + + # Pull the freqtrade image + docker-compose pull + + # Create user directory structure + docker-compose run --rm freqtrade create-userdir --userdir user_data + + # Create configuration - Requires answering interactive questions + docker-compose run --rm freqtrade new-config --config user_data/config.json + ``` + +=== "RaspberryPi" + ``` bash + mkdir ft_userdata + cd ft_userdata/ + # Download the docker-compose file from the repository + curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml + + # Pull the freqtrade image + docker-compose pull + + # Create user directory structure + docker-compose run --rm freqtrade create-userdir --userdir user_data + + # Create configuration - Requires answering interactive questions + docker-compose run --rm freqtrade new-config --config user_data/config.json + ``` + + !!! Note "Change your docker Image" + You have to change the docker image in the docker-compose file for your Raspberry build to work properly. + ``` yml + image: freqtradeorg/freqtrade:stable_pi + # image: freqtradeorg/freqtrade:develop_pi + ``` + +The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image. +The last 2 steps in the snippet create the directory with `user_data`, as well as (interactively) the default configuration based on your selections. + +!!! Question "How to edit the bot configuration?" + You can edit the configuration at any time, which is available as `user_data/config.json` (within the directory `ft_userdata`) when using the above configuration. + + You can also change the both Strategy and commands by editing the `docker-compose.yml` file. + +#### Adding a custom strategy + +1. The configuration is now available as `user_data/config.json` +2. Copy a custom strategy to the directory `user_data/strategies/` +3. add the Strategy' class name to the `docker-compose.yml` file + +The `SampleStrategy` is run by default. + +!!! Warning "`SampleStrategy` is just a demo!" + The `SampleStrategy` is there for your reference and give you ideas for your own strategy. + Please always backtest the strategy and use dry-run for some time before risking real money! + +Once this is done, you're ready to launch the bot in trading mode (Dry-run or Live-trading, depending on your answer to the corresponding question you made above). + +``` bash +docker-compose up -d +``` + +#### Docker-compose logs + +Logs will be located at: `user_data/logs/freqtrade.log`. +You can check the latest log with the command `docker-compose logs -f`. + +#### Database + +The database will be at: `user_data/tradesv3.sqlite` + +#### Updating freqtrade with docker-compose + +To update freqtrade when using `docker-compose` is as simple as running the following 2 commands: + +``` bash +# Download the latest image +docker-compose pull +# Restart the image +docker-compose up -d +``` + +This will first pull the latest image, and will then restart the container with the just pulled version. + +!!! Warning "Check the Changelog" + You should always check the changelog for breaking changes / manual interventions required and make sure the bot starts correctly after the update. + +### Editing the docker-compose file + +Advanced users may edit the docker-compose file further to include all possible options or arguments. + +All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. + +!!! Note "`docker-compose run --rm`" + Including `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). + +#### Example: Download data with docker-compose + +Download backtesting data for 5 days for the pair ETH/BTC and 1h timeframe from Binance. The data will be stored in the directory `user_data/data/` on the host. + +``` bash +docker-compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h +``` + +Head over to the [Data Downloading Documentation](data-download.md) for more details on downloading data. + +#### Example: Backtest with docker-compose + +Run backtesting in docker-containers for SampleStrategy and specified timerange of historical data, on 5m timeframe: + +``` bash +docker-compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m +``` + +Head over to the [Backtesting Documentation](backtesting.md) to learn more. + +### Additional dependencies with docker-compose + +If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host. +For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) for an example). + +You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions. + +``` yaml + image: freqtrade_custom + build: + context: . + dockerfile: "./Dockerfile." +``` + +You can then run `docker-compose build` to build the docker image, and run it using the commands described above. + +## Plotting with docker-compose + +Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file. +You can then use these commands as follows: + +``` bash +docker-compose run --rm freqtrade plot-dataframe --strategy AwesomeStrategy -p BTC/ETH --timerange=20180801-20180805 +``` + +The output will be stored in the `user_data/plot` directory, and can be opened with any modern browser. + +## Data analayis using docker compose + +Freqtrade provides a docker-compose file which starts up a jupyter lab server. +You can run this server using the following command: + +``` bash +docker-compose --rm -f docker/docker-compose-jupyter.yml up +``` + +This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`. +Please use the link that's printed in the console after startup for simplified login. + +Since part of this image is built on your machine, it is recommended to rebuild the image from time to time to keep freqtrade (and dependencies) uptodate. + +``` bash +docker-compose -f docker/docker-compose-jupyter.yml build --no-cache +``` diff --git a/docs/edge.md b/docs/edge.md index dcb559f96..500c3c833 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1,92 +1,142 @@ # Edge positioning -This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss. +The `Edge Positioning` module uses probability to calculate your win rate and risk reward ration. It will use these statistics to control your strategy trade entry points, position side and, stoploss. !!! Warning - Edge positioning is not compatible with dynamic (volume-based) whitelist. + `Edge positioning` is not compatible with dynamic (volume-based) whitelist. !!! Note - Edge does not consider anything other than *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file. - Therefore, it is important to understand that Edge can improve the performance of some trading strategies but *decrease* the performance of others. + `Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file. + `Edge Positioning` improves the performance of some trading strategies and *decreases* the performance of others. ## Introduction -Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose. +Trading strategies are not perfect. They are frameworks that are susceptible to the market and its indicators. Because the market is not at all predictable, sometimes a strategy will win and sometimes the same strategy will lose. -But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: you give me 10$. Is it an interesting game? No, it's quite boring, isn't it? +To obtain an edge in the market, a strategy has to make more money than it loses. Making money in trading is not only about *how often* the strategy makes or loses money. -But let's say the probability that we have heads is 80% (because our coin has the displaced distribution of mass or other defect), and the probability that we have tails is 20%. Now it is becoming interesting... +!!! tip "It doesn't matter how often, but how much!" + A bad strategy might make 1 penny in *ten* transactions but lose 1 dollar in *one* transaction. If one only checks the number of winning trades, it would be misleading to think that the strategy is actually making a profit. -That means 10$ X 80% versus 10$ X 20%. 8$ versus 2$. That means over time you will win 8$ risking only 2$ on each toss of coin. +The Edge Positioning module seeks to improve a strategy's winning probability and the money that the strategy will make *on the long run*. -Let's complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. The calculation is: 80% X 2$ versus 20% X 8$. It is becoming boring again because overtime you win $1.6$ (80% X 2$) and me $1.6 (20% X 8$) too. +We raise the following question[^1]: -The question is: How do you calculate that? How do you know if you wanna play? +!!! Question "Which trade is a better option?" + a) A trade with 80% of chance of losing $100 and 20% chance of winning $200
+ b) A trade with 100% of chance of losing $30 -The answer comes to two factors: +???+ Info "Answer" + The expected value of *a)* is smaller than the expected value of *b)*.
+ Hence, *b*) represents a smaller loss in the long run.
+ However, the answer is: *it depends* -- Win Rate -- Risk Reward Ratio +Another way to look at it is to ask a similar question: -### Win Rate +!!! Question "Which trade is a better option?" + a) A trade with 80% of chance of winning 100 and 20% chance of losing $200
+ b) A trade with 100% of chance of winning $30 -Win Rate (*W*) is is the mean over some amount of trades (*N*) what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only if you won or not). +Edge positioning tries to answer the hard questions about risk/reward and position size automatically, seeking to minimizes the chances of losing of a given strategy. -``` -W = (Number of winning trades) / (Total number of trades) = (Number of winning trades) / N -``` +### Trading, winning and losing -Complementary Loss Rate (*L*) is defined as +Let's call $o$ the return of a single transaction $o$ where $o \in \mathbb{R}$. The collection $O = \{o_1, o_2, ..., o_N\}$ is the set of all returns of transactions made during a trading session. We say that $N$ is the cardinality of $O$, or, in lay terms, it is the number of transactions made in a trading session. -``` -L = (Number of losing trades) / (Total number of trades) = (Number of losing trades) / N -``` +!!! Example + In a session where a strategy made three transactions we can say that $O = \{3.5, -1, 15\}$. That means that $N = 3$ and $o_1 = 3.5$, $o_2 = -1$, $o_3 = 15$. -or, which is the same, as +A winning trade is a trade where a strategy *made* money. Making money means that the strategy closed the position in a value that returned a profit, after all deducted fees. Formally, a winning trade will have a return $o_i > 0$. Similarly, a losing trade will have a return $o_j \leq 0$. With that, we can discover the set of all winning trades, $T_{win}$, as follows: -``` -L = 1 – W -``` +$$ T_{win} = \{ o \in O | o > 0 \} $$ + +Similarly, we can discover the set of losing trades $T_{lose}$ as follows: + +$$ T_{lose} = \{o \in O | o \leq 0\} $$ + +!!! Example + In a section where a strategy made three transactions $O = \{3.5, -1, 15, 0\}$:
+ $T_{win} = \{3.5, 15\}$
+ $T_{lose} = \{-1, 0\}$
+ +### Win Rate and Lose Rate + +The win rate $W$ is the proportion of winning trades with respect to all the trades made by a strategy. We use the following function to compute the win rate: + +$$W = \frac{|T_{win}|}{N}$$ + +Where $W$ is the win rate, $N$ is the number of trades and, $T_{win}$ is the set of all trades where the strategy made money. + +Similarly, we can compute the rate of losing trades: + +$$ + L = \frac{|T_{lose}|}{N} +$$ + +Where $L$ is the lose rate, $N$ is the amount of trades made and, $T_{lose}$ is the set of all trades where the strategy lost money. Note that the above formula is the same as calculating $L = 1 – W$ or $W = 1 – L$ ### Risk Reward Ratio -Risk Reward Ratio (*R*) is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose: +Risk Reward Ratio ($R$) is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose. Formally: -``` -R = Profit / Loss -``` +$$ R = \frac{\text{potential_profit}}{\text{potential_loss}} $$ -Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: +???+ Example "Worked example of $R$ calculation" + Let's say that you think that the price of *stonecoin* today is $10.0. You believe that, because they will start mining stonecoin, it will go up to $15.0 tomorrow. There is the risk that the stone is too hard, and the GPUs can't mine it, so the price might go to $0 tomorrow. You are planning to invest $100.
+ Your potential profit is calculated as:
+ $\begin{aligned} + \text{potential_profit} &= (\text{potential_price} - \text{cost_per_unit}) * \frac{\text{investment}}{\text{cost_per_unit}} \\ + &= (15 - 10) * \frac{100}{15}\\ + &= 33.33 + \end{aligned}$
+ Since the price might go to $0, the $100 dolars invested could turn into 0. We can compute the Risk Reward Ratio as follows:
+ $\begin{aligned} + R &= \frac{\text{potential_profit}}{\text{potential_loss}}\\ + &= \frac{33.33}{100}\\ + &= 0.333... + \end{aligned}$
+ What it effectivelly means is that the strategy have the potential to make $0.33 for each $1 invested. -``` -Average profit = (Sum of profits) / (Number of winning trades) +On a long horizon, that is, on many trades, we can calculate the risk reward by dividing the strategy' average profit on winning trades by the strategy' average loss on losing trades. We can calculate the average profit, $\mu_{win}$, as follows: -Average loss = (Sum of losses) / (Number of losing trades) +$$ \text{average_profit} = \mu_{win} = \frac{\text{sum_of_profits}}{\text{count_winning_trades}} = \frac{\sum^{o \in T_{win}} o}{|T_{win}|} $$ -R = (Average profit) / (Average loss) -``` +Similarly, we can calculate the average loss, $\mu_{lose}$, as follows: + +$$ \text{average_loss} = \mu_{lose} = \frac{\text{sum_of_losses}}{\text{count_losing_trades}} = \frac{\sum^{o \in T_{lose}} o}{|T_{lose}|} $$ + +Finally, we can calculate the Risk Reward ratio, $R$, as follows: + +$$ R = \frac{\text{average_profit}}{\text{average_loss}} = \frac{\mu_{win}}{\mu_{lose}}\\ $$ + + +???+ Example "Worked example of $R$ calculation using mean profit/loss" + Let's say the strategy that we are using makes an average win $\mu_{win} = 2.06$ and an average loss $\mu_{loss} = 4.11$.
+ We calculate the risk reward ratio as follows:
+ $R = \frac{\mu_{win}}{\mu_{loss}} = \frac{2.06}{4.11} = 0.5012...$ + ### Expectancy -At this point we can combine *W* and *R* to create an expectancy ratio. This is a simple process of multiplying the risk reward ratio by the percentage of winning trades and subtracting the percentage of losing trades, which is calculated as follows: +By combining the Win Rate $W$ and and the Risk Reward ratio $R$ to create an expectancy ratio $E$. A expectance ratio is the expected return of the investment made in a trade. We can compute the value of $E$ as follows: -``` -Expectancy Ratio = (Risk Reward Ratio X Win Rate) – Loss Rate = (R X W) – L -``` +$$E = R * W - L$$ -So lets say your Win rate is 28% and your Risk Reward Ratio is 5: +!!! Example "Calculating $E$" + Let's say that a strategy has a win rate $W = 0.28$ and a risk reward ratio $R = 5$. What this means is that the strategy is expected to make 5 times the investment around on 28% of the trades it makes. Working out the example:
+ $E = R * W - L = 5 * 0.28 - 0.72 = 0.68$ +
-``` -Expectancy = (5 X 0.28) – 0.72 = 0.68 -``` +The expectancy worked out in the example above means that, on average, this strategy' trades will return 1.68 times the size of its losses. Said another way, the strategy makes $1.68 for every $1 it loses, on average. -Superficially, this means that on average you expect this strategy’s trades to return 1.68 times the size of your loses. Said another way, you can expect to win $1.68 for every $1 you lose. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. +This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future. You can also use this value to evaluate the effectiveness of modifications to this system. -**NOTICE:** It's important to keep in mind that Edge is testing your expectancy using historical data, there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology, but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. +!!! Note + It's important to keep in mind that Edge is testing your expectancy using historical data, there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. ## How does it work? @@ -99,13 +149,13 @@ Edge combines dynamic stoploss, dynamic positions, and whitelist generation into | XZC/ETH | -0.03 | 0.52 |1.359670 | 0.228 | | XZC/ETH | -0.04 | 0.51 |1.234539 | 0.117 | -The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. In the above example stoploss at 3% leads to the maximum expectancy according to historical data. +The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. In the above example stoploss at $3%$ leads to the maximum expectancy according to historical data. Edge module then forces stoploss value it evaluated to your strategy dynamically. ### Position size -Edge also dictates the stake amount for each trade to the bot according to the following factors: +Edge dictates the amount at stake for each trade to the bot according to the following factors: - Allowed capital at risk - Stoploss @@ -116,9 +166,9 @@ Allowed capital at risk is calculated as follows: Allowed capital at risk = (Capital available_percentage) X (Allowed risk per trade) ``` -Stoploss is calculated as described above against historical data. +Stoploss is calculated as described above with respect to historical data. -Your position size then will be: +The position size is calculated as follows: ``` Position size = (Allowed capital at risk) / Stoploss @@ -126,19 +176,23 @@ Position size = (Allowed capital at risk) / Stoploss Example: -Let's say the stake currency is ETH and you have 10 ETH on the exchange, your capital available percentage is 50% and you would allow 1% of risk for each trade. thus your available capital for trading is **10 x 0.5 = 5 ETH** and allowed capital at risk would be **5 x 0.01 = 0.05 ETH**. +Let's say the stake currency is **ETH** and there is $10$ **ETH** on the wallet. The capital available percentage is $50%$ and the allowed risk per trade is $1\%$. Thus, the available capital for trading is $10 * 0.5 = 5$ **ETH** and the allowed capital at risk would be $5 * 0.01 = 0.05$ **ETH**. -Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.05 / 0.02 = 2.5 ETH**. +- **Trade 1:** The strategy detects a new buy signal in the **XLM/ETH** market. `Edge Positioning` calculates a stoploss of $2\%$ and a position of $0.05 / 0.02 = 2.5$ **ETH**. The bot takes a position of $2.5$ **ETH** in the **XLM/ETH** market. -Bot takes a position of 2.5 ETH on XLM/ETH (call it trade 1). Up next, you receive another buy signal while trade 1 is still open. This time on **BTC/ETH** market. Edge calculated stoploss for this market at 4%. So your position size would be 0.05 / 0.04 = 1.25 ETH (call it trade 2). +- **Trade 2:** The strategy detects a buy signal on the **BTC/ETH** market while **Trade 1** is still open. `Edge Positioning` calculates the stoploss of $4\%$ on this market. Thus, **Trade 2** position size is $0.05 / 0.04 = 1.25$ **ETH**. -Note that available capital for trading didn’t change for trade 2 even if you had already trade 1. The available capital doesn’t mean the free amount on your wallet. +!!! Tip "Available Capital $\neq$ Available in wallet" + The available capital for trading didn't change in **Trade 2** even with **Trade 1** still open. The available capital **is not** the free amount in the wallet. -Now you have two trades open. The bot receives yet another buy signal for another market: **ADA/ETH**. This time the stoploss is calculated at 1%. So your position size is **0.05 / 0.01 = 5 ETH**. But there are already 3.75 ETH blocked in two previous trades. So the position size for this third trade would be **5 – 3.75 = 1.25 ETH**. +- **Trade 3:** The strategy detects a buy signal in the **ADA/ETH** market. `Edge Positioning` calculates a stoploss of $1\%$ and a position of $0.05 / 0.01 = 5$ **ETH**. Since **Trade 1** has $2.5$ **ETH** blocked and **Trade 2** has $1.25$ **ETH** blocked, there is only $5 - 1.25 - 2.5 = 1.25$ **ETH** available. Hence, the position size of **Trade 3** is $1.25$ **ETH**. -Available capital doesn’t change before a position is sold. Let’s assume that trade 1 receives a sell signal and it is sold with a profit of 1 ETH. Your total capital on exchange would be 11 ETH and the available capital for trading becomes 5.5 ETH. +!!! Tip "Available Capital Updates" + The available capital does not change before a position is sold. After a trade is closed the Available Capital goes up if the trade was profitable or goes down if the trade was a loss. -So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be **0.055 / 0.02 = 2.75 ETH**. +- The strategy detects a sell signal in the **XLM/ETH** market. The bot exits **Trade 1** for a profit of $1$ **ETH**. The total capital in the wallet becomes $11$ **ETH** and the available capital for trading becomes $5.5$ **ETH**. + +- **Trade 4** The strategy detects a new buy signal int the **XLM/ETH** market. `Edge Positioning` calculates the stoploss of $2%$, and the position size of $0.055 / 0.02 = 2.75$ **ETH**. ## Configurations @@ -169,23 +223,23 @@ freqtrade edge An example of its output: -| pair | stoploss | win rate | risk reward ratio | required risk reward | expectancy | total number of trades | average duration (min) | -|:----------|-----------:|-----------:|--------------------:|-----------------------:|-------------:|-------------------------:|-------------------------:| -| AGI/BTC | -0.02 | 0.64 | 5.86 | 0.56 | 3.41 | 14 | 54 | -| NXS/BTC | -0.03 | 0.64 | 2.99 | 0.57 | 1.54 | 11 | 26 | -| LEND/BTC | -0.02 | 0.82 | 2.05 | 0.22 | 1.50 | 11 | 36 | -| VIA/BTC | -0.01 | 0.55 | 3.01 | 0.83 | 1.19 | 11 | 48 | -| MTH/BTC | -0.09 | 0.56 | 2.82 | 0.80 | 1.12 | 18 | 52 | -| ARDR/BTC | -0.04 | 0.42 | 3.14 | 1.40 | 0.73 | 12 | 42 | -| BCPT/BTC | -0.01 | 0.71 | 1.34 | 0.40 | 0.67 | 14 | 30 | -| WINGS/BTC | -0.02 | 0.56 | 1.97 | 0.80 | 0.65 | 27 | 42 | -| VIBE/BTC | -0.02 | 0.83 | 0.91 | 0.20 | 0.59 | 12 | 35 | -| MCO/BTC | -0.02 | 0.79 | 0.97 | 0.27 | 0.55 | 14 | 31 | -| GNT/BTC | -0.02 | 0.50 | 2.06 | 1.00 | 0.53 | 18 | 24 | -| HOT/BTC | -0.01 | 0.17 | 7.72 | 4.81 | 0.50 | 209 | 7 | -| SNM/BTC | -0.03 | 0.71 | 1.06 | 0.42 | 0.45 | 17 | 38 | -| APPC/BTC | -0.02 | 0.44 | 2.28 | 1.27 | 0.44 | 25 | 43 | -| NEBL/BTC | -0.03 | 0.63 | 1.29 | 0.58 | 0.44 | 19 | 59 | +| **pair** | **stoploss** | **win rate** | **risk reward ratio** | **required risk reward** | **expectancy** | **total number of trades** | **average duration (min)** | +|:----------|-----------:|-----------:|--------------------:|-----------------------:|-------------:|-----------------:|---------------:| +| **AGI/BTC** | -0.02 | 0.64 | 5.86 | 0.56 | 3.41 | 14 | 54 | +| **NXS/BTC** | -0.03 | 0.64 | 2.99 | 0.57 | 1.54 | 11 | 26 | +| **LEND/BTC** | -0.02 | 0.82 | 2.05 | 0.22 | 1.50 | 11 | 36 | +| **VIA/BTC** | -0.01 | 0.55 | 3.01 | 0.83 | 1.19 | 11 | 48 | +| **MTH/BTC** | -0.09 | 0.56 | 2.82 | 0.80 | 1.12 | 18 | 52 | +| **ARDR/BTC** | -0.04 | 0.42 | 3.14 | 1.40 | 0.73 | 12 | 42 | +| **BCPT/BTC** | -0.01 | 0.71 | 1.34 | 0.40 | 0.67 | 14 | 30 | +| **WINGS/BTC** | -0.02 | 0.56 | 1.97 | 0.80 | 0.65 | 27 | 42 | +| **VIBE/BTC** | -0.02 | 0.83 | 0.91 | 0.20 | 0.59 | 12 | 35 | +| **MCO/BTC** | -0.02 | 0.79 | 0.97 | 0.27 | 0.55 | 14 | 31 | +| **GNT/BTC** | -0.02 | 0.50 | 2.06 | 1.00 | 0.53 | 18 | 24 | +| **HOT/BTC** | -0.01 | 0.17 | 7.72 | 4.81 | 0.50 | 209 | 7 | +| **SNM/BTC** | -0.03 | 0.71 | 1.06 | 0.42 | 0.45 | 17 | 38 | +| **APPC/BTC** | -0.02 | 0.44 | 2.28 | 1.27 | 0.44 | 25 | 43 | +| **NEBL/BTC** | -0.03 | 0.63 | 1.29 | 0.58 | 0.44 | 19 | 59 | Edge produced the above table by comparing `calculate_since_number_of_days` to `minimum_expectancy` to find `min_trade_number` historical information based on the config file. The timerange Edge uses for its comparisons can be further limited by using the `--timerange` switch. @@ -218,3 +272,6 @@ The full timerange specification: * Use tickframes since 2018/01/31: `--timerange=20180131-` * Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` * Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` + + +[^1]: Question extracted from MIT Opencourseware S096 - Mathematics with applications in Finance: https://ocw.mit.edu/courses/mathematics/18-s096-topics-in-mathematics-with-applications-in-finance-fall-2013/ diff --git a/docs/faq.md b/docs/faq.md index 48f52a566..beed89801 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -89,7 +89,7 @@ Same fix should be done in the configuration file, if order types are defined in ### How do I search the bot logs for something? -By default, the bot writes its log into stderr stream. This is implemented this way so that you can easily separate the bot's diagnostics messages from Backtesting, Edge and Hyperopt results, output from other various Freqtrade utility subcommands, as well as from the output of your custom `print()`'s you may have inserted into your strategy. So if you need to search the log messages with the grep utility, you need to redirect stderr to stdout and disregard stdout. +By default, the bot writes its log into stderr stream. This is implemented this way so that you can easily separate the bot's diagnostics messages from Backtesting, Edge and Hyperopt results, output from other various Freqtrade utility sub-commands, as well as from the output of your custom `print()`'s you may have inserted into your strategy. So if you need to search the log messages with the grep utility, you need to redirect stderr to stdout and disregard stdout. * In unix shells, this normally can be done as simple as: ```shell @@ -114,7 +114,7 @@ and then grep it as: ```shell $ cat /path/to/mylogfile.log | grep 'something' ``` -or even on the fly, as the bot works and the logfile grows: +or even on the fly, as the bot works and the log file grows: ```shell $ tail -f /path/to/mylogfile.log | grep 'something' ``` @@ -137,7 +137,7 @@ compute. Since hyperopt uses Bayesian search, running for too many epochs may not produce greater results. -It's therefore recommended to run between 500-1000 epochs over and over until you hit at least 10.000 epocs in total (or are satisfied with the result). You can best judge by looking at the results - if the bot keeps discovering better strategies, it's best to keep on going. +It's therefore recommended to run between 500-1000 epochs over and over until you hit at least 10.000 epochs in total (or are satisfied with the result). You can best judge by looking at the results - if the bot keeps discovering better strategies, it's best to keep on going. ```bash freqtrade hyperopt -e 1000 @@ -153,7 +153,7 @@ for i in {1..100}; do freqtrade hyperopt -e 1000; done * Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. -* If you wonder why it can take from 20 minutes to days to do 1000 epocs here are some answers: +* If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers: This answer was written during the release 0.15.1, when we had: @@ -167,7 +167,7 @@ already 8\*10^9\*10 evaluations. A roughly total of 80 billion evals. Did you run 100 000 evals? Congrats, you've done roughly 1 / 100 000 th of the search space, assuming that the bot never tests the same parameters more than once. -* The time it takes to run 1000 hyperopt epocs depends on things like: The available cpu, harddisk, ram, timeframe, timerange, indicator settings, indicator count, amount of coins that hyperopt test strategies on and the resulting trade count - which can be 650 trades in a year or 10.0000 trades depending if the strategy aims for big profits by trading rarely or for many low profit trades. +* The time it takes to run 1000 hyperopt epochs depends on things like: The available cpu, hard-disk, ram, timeframe, timerange, indicator settings, indicator count, amount of coins that hyperopt test strategies on and the resulting trade count - which can be 650 trades in a year or 10.0000 trades depending if the strategy aims for big profits by trading rarely or for many low profit trades. Example: 4% profit 650 times vs 0,3% profit a trade 10.000 times in a year. If we assume you set the --timerange to 365 days. @@ -180,7 +180,7 @@ Example: The Edge module is mostly a result of brainstorming of [@mishaker](https://github.com/mishaker) and [@creslinux](https://github.com/creslinux) freqtrade team members. -You can find further info on expectancy, winrate, risk management and position size in the following sources: +You can find further info on expectancy, win rate, risk management and position size in the following sources: - https://www.tradeciety.com/ultimate-math-guide-for-traders/ - http://www.vantharp.com/tharp-concepts/expectancy.asp diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 530faf700..3f7a27ef0 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -229,7 +229,7 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -freqtrade hyperopt --config config.json --hyperopt -e 5000 --spaces all +freqtrade hyperopt --config config.json --hyperopt -e 500 --spaces all ``` Use `` as the name of the custom hyperopt used. diff --git a/docs/index.md b/docs/index.md index adc661300..e7fc54628 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,7 @@ Fork -Download +Download Follow @freqtrade @@ -37,13 +37,9 @@ Freqtrade is a crypto-currency algorithmic trading software developed in python ## Requirements -### Up to date clock - -The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges. - ### Hardware requirements -To run this bot we recommend you a cloud instance with a minimum of: +To run this bot we recommend you a linux cloud instance with a minimum of: - 2GB RAM - 1GB disk space diff --git a/docs/installation.md b/docs/installation.md index c03be55d1..9b15c9685 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -18,6 +18,9 @@ Click each one for install guide: We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot), which is optional but recommended. +!!! Warning "Up-to-date clock" + The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges. + ## Quick start Freqtrade provides the Linux/MacOS Easy Installation script to install all dependencies and help you configure the bot. @@ -28,7 +31,7 @@ Freqtrade provides the Linux/MacOS Easy Installation script to install all depen The easiest way to install and run Freqtrade is to clone the bot Github repository and then run the Easy Installation script, if it's available for your platform. !!! Note "Version considerations" - When cloning the repository the default working branch has the name `develop`. This branch contains all last features (can be considered as relatively stable, thanks to automated tests). The `master` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable). + When cloning the repository the default working branch has the name `develop`. This branch contains all last features (can be considered as relatively stable, thanks to automated tests). The `stable` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable). !!! Note Python3.6 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. @@ -38,11 +41,11 @@ This can be achieved with the following commands: ```bash git clone https://github.com/freqtrade/freqtrade.git cd freqtrade -git checkout master # Optional, see (1) +# git checkout stable # Optional, see (1) ./setup.sh --install ``` -(1) This command switches the cloned repository to the use of the `master` branch. It's not needed if you wish to stay on the `develop` branch. You may later switch between branches at any time with the `git checkout master`/`git checkout develop` commands. +(1) This command switches the cloned repository to the use of the `stable` branch. It's not needed if you wish to stay on the `develop` branch. You may later switch between branches at any time with the `git checkout stable`/`git checkout develop` commands. ## Easy Installation Script (Linux/MacOS) @@ -53,7 +56,7 @@ $ ./setup.sh usage: -i,--install Install freqtrade from scratch -u,--update Command git pull to update. - -r,--reset Hard reset your develop/master branch. + -r,--reset Hard reset your develop/stable branch. -c,--config Easy config generator (Will override your existing file). ``` @@ -73,12 +76,16 @@ This option will pull the last version of your current branch and update your vi ** --reset ** -This option will hard reset your branch (only if you are on either `master` or `develop`) and recreate your virtualenv. +This option will hard reset your branch (only if you are on either `stable` or `develop`) and recreate your virtualenv. ** --config ** DEPRECATED - use `freqtrade new-config -c config.json` instead. +### Activate your virtual environment + +Each time you open a new terminal, you must run `source .env/bin/activate`. + ------ ## Custom Installation @@ -89,36 +96,34 @@ OS Specific steps are listed first, the [Common](#common) section below is neces !!! Note Python3.6 or higher and the corresponding pip are assumed to be available. -### Linux - Ubuntu 16.04 +=== "Ubuntu 16.04" + #### Install necessary dependencies -#### Install necessary dependencies + ```bash + sudo apt-get update + sudo apt-get install build-essential git + ``` -```bash -sudo apt-get update -sudo apt-get install build-essential git -``` +=== "RaspberryPi/Raspbian" + The following assumes the latest [Raspbian Buster lite image](https://www.raspberrypi.org/downloads/raspbian/) from at least September 2019. + This image comes with python3.7 preinstalled, making it easy to get freqtrade up and running. -### Raspberry Pi / Raspbian + Tested using a Raspberry Pi 3 with the Raspbian Buster lite image, all updates applied. -The following assumes the latest [Raspbian Buster lite image](https://www.raspberrypi.org/downloads/raspbian/) from at least September 2019. -This image comes with python3.7 preinstalled, making it easy to get freqtrade up and running. + ``` bash + sudo apt-get install python3-venv libatlas-base-dev + git clone https://github.com/freqtrade/freqtrade.git + cd freqtrade -Tested using a Raspberry Pi 3 with the Raspbian Buster lite image, all updates applied. + bash setup.sh -i + ``` -``` bash -sudo apt-get install python3-venv libatlas-base-dev -git clone https://github.com/freqtrade/freqtrade.git -cd freqtrade + !!! Note "Installation duration" + Depending on your internet speed and the Raspberry Pi version, installation can take multiple hours to complete. -bash setup.sh -i -``` - -!!! Note "Installation duration" - Depending on your internet speed and the Raspberry Pi version, installation can take multiple hours to complete. - -!!! Note - The above does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. - We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerful machine. + !!! Note + The above does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. + We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerful machine. ### Common @@ -169,12 +174,7 @@ Clone the git repository: ```bash git clone https://github.com/freqtrade/freqtrade.git cd freqtrade -``` - -Optionally checkout the master branch to get the latest stable release: - -```bash -git checkout master +git checkout stable ``` #### 4. Install python dependencies @@ -212,73 +212,19 @@ On Linux, as an optional post-installation task, you may wish to setup the bot t ------ -## Using Conda +### Anaconda Freqtrade can also be installed using Anaconda (or Miniconda). +!!! Note + This requires the [ta-lib](#1-install-ta-lib) C-library to be installed first. See below. + ``` bash conda env create -f environment.yml ``` -!!! Note - This requires the [ta-lib](#1-install-ta-lib) C-library to be installed first. - -## Windows - -We recommend that Windows users use [Docker](docker.md) as this will work much easier and smoother (also more secure). - -If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work. -If that is not available on your system, feel free to try the instructions below, which led to success for some. - -### Install freqtrade manually - -!!! Note - Make sure to use 64bit Windows and 64bit Python to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows. - -!!! Hint - Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Conda section](#using-conda) in this document for more information. - -#### Clone the git repository - -```bash -git clone https://github.com/freqtrade/freqtrade.git -``` - -#### Install ta-lib - -Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). - -As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version) - -```cmd ->cd \path\freqtrade-develop ->python -m venv .env ->.env\Scripts\activate.bat -REM optionally install ta-lib from wheel -REM >pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl ->pip install -r requirements.txt ->pip install -e . ->freqtrade -``` - -> Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222) - -#### Error during installation under Windows - -``` bash -error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools -``` - -Unfortunately, many packages requiring compilation don't provide a pre-build wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use. - -The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building c code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or [docker](docker.md) first. - ---- - -Now you have an environment ready, the next step is -[Bot Configuration](configuration.md). - -## Troubleshooting +----- +## Troubleshooting ### MacOS installation error @@ -291,4 +237,9 @@ For MacOS 10.14, this can be accomplished with the below command. open /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg ``` -If this file is inexistant, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details. +If this file is inexistent, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details. + +----- + +Now you have an environment ready, the next step is +[Bot Configuration](configuration.md). diff --git a/docs/javascripts/config.js b/docs/javascripts/config.js new file mode 100644 index 000000000..95d619efc --- /dev/null +++ b/docs/javascripts/config.js @@ -0,0 +1,12 @@ +window.MathJax = { + tex: { + inlineMath: [["\\(", "\\)"]], + displayMath: [["\\[", "\\]"]], + processEscapes: true, + processEnvironments: true + }, + options: { + ignoreHtmlClass: ".*|", + processHtmlClass: "arithmatex" + } +}; \ No newline at end of file diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 5226db750..d4c93928e 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,2 +1,2 @@ -mkdocs-material==5.5.8 +mkdocs-material==5.5.13 mdx_truly_sane_lists==1.2 diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index cf785ced6..168d416ab 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -46,7 +46,7 @@ sqlite3 ### Trade table structure ```sql -CREATE TABLE trades +CREATE TABLE trades( id INTEGER NOT NULL, exchange VARCHAR NOT NULL, pair VARCHAR NOT NULL, diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index be08faa2d..14d5fcd84 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -483,6 +483,9 @@ if self.dp: ### Complete Data-provider sample ```python +from freqtrade.strategy import IStrategy, merge_informative_pair +from pandas import DataFrame + class SampleStrategy(IStrategy): # strategy init stuff... @@ -513,17 +516,12 @@ class SampleStrategy(IStrategy): # Get the 14 day rsi informative['rsi'] = ta.RSI(informative, timeperiod=14) - # Rename columns to be unique - informative.columns = [f"{col}_{inf_tf}" for col in informative.columns] - # Assuming inf_tf = '1d' - then the columns will now be: - # date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d - - # Combine the 2 dataframes - # all indicators on the informative sample MUST be calculated before this point - dataframe = pd.merge(dataframe, informative, left_on='date', right_on=f'date_{inf_tf}', how='left') - # FFill to have the 1d value available in every row throughout the day. - # Without this, comparisons would only work once per day. - dataframe = dataframe.ffill() + # Use the helper function merge_informative_pair to safely merge the pair + # Automatically renames the columns and merges a shorter timeframe dataframe and a longer timeframe informative pair + # use ffill to have the 1d value available in every row throughout the day. + # Without this, comparisons between columns of the original and the informative pair would only work once per day. + # Full documentation of this method, see below + dataframe = merge_informative_pair(dataframe, informative, self.timeframe, inf_tf, ffill=True) # Calculate rsi of the original dataframe (5m timeframe) dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) @@ -547,6 +545,69 @@ class SampleStrategy(IStrategy): *** +## Helper functions + +### *merge_informative_pair()* + +This method helps you merge an informative pair to a regular dataframe without lookahead bias. +It's there to help you merge the dataframe in a safe and consistent way. + +Options: + +- Rename the columns for you to create unique columns +- Merge the dataframe without lookahead bias +- Forward-fill (optional) + +All columns of the informative dataframe will be available on the returning dataframe in a renamed fashion: + +!!! Example "Column renaming" + Assuming `inf_tf = '1d'` the resulting columns will be: + + ``` python + 'date', 'open', 'high', 'low', 'close', 'rsi' # from the original dataframe + 'date_1d', 'open_1d', 'high_1d', 'low_1d', 'close_1d', 'rsi_1d' # from the informative dataframe + ``` + +??? Example "Column renaming - 1h" + Assuming `inf_tf = '1h'` the resulting columns will be: + + ``` python + 'date', 'open', 'high', 'low', 'close', 'rsi' # from the original dataframe + 'date_1h', 'open_1h', 'high_1h', 'low_1h', 'close_1h', 'rsi_1h' # from the informative dataframe + ``` + +??? Example "Custom implementation" + A custom implementation for this is possible, and can be done as follows: + + ``` python + + # Shift date by 1 candle + # This is necessary since the data is always the "open date" + # and a 15m candle starting at 12:15 should not know the close of the 1h candle from 12:00 to 13:00 + minutes = timeframe_to_minutes(inf_tf) + # Only do this if the timeframes are different: + informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes, 'm') + + # Rename columns to be unique + informative.columns = [f"{col}_{inf_tf}" for col in informative.columns] + # Assuming inf_tf = '1d' - then the columns will now be: + # date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d + + # Combine the 2 dataframes + # all indicators on the informative sample MUST be calculated before this point + dataframe = pd.merge(dataframe, informative, left_on='date', right_on=f'date_merge_{inf_tf}', how='left') + # FFill to have the 1d value available in every row throughout the day. + # Without this, comparisons would only work once per day. + dataframe = dataframe.ffill() + + ``` + +!!! Warning "Informative timeframe < timeframe" + Using informative timeframes smaller than the dataframe timeframe is not recommended with this method, as it will not use any of the additional information this would provide. + To use the more detailed information properly, more advanced methods should be applied (which are out of scope for freqtrade documentation, as it'll depend on the respective need). + +*** + ## Additional data (Wallets) The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 5f804386d..ce2d715a0 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -41,6 +41,34 @@ Talk to the [userinfobot](https://telegram.me/userinfobot) Get your "Id", you will use it for the config parameter `chat_id`. +## Control telegram noise + +Freqtrade provides means to control the verbosity of your telegram bot. +Each setting has the following possible values: + +* `on` - Messages will be sent, and user will be notified. +* `silent` - Message will be sent, Notification will be without sound / vibration. +* `off` - Skip sending a message-type all together. + +Example configuration showing the different settings: + +``` json +"telegram": { + "enabled": true, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id", + "notification_settings": { + "status": "silent", + "warning": "on", + "startup": "off", + "buy": "silent", + "sell": "on", + "buy_cancel": "silent", + "sell_cancel": "on" + } + }, +``` + ## Telegram commands Per default, the Telegram bot shows predefined commands. Some commands diff --git a/docs/windows_installation.md b/docs/windows_installation.md new file mode 100644 index 000000000..f7900d85a --- /dev/null +++ b/docs/windows_installation.md @@ -0,0 +1,57 @@ +We **strongly** recommend that Windows users use [Docker](docker.md) as this will work much easier and smoother (also more secure). + +If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work. +Otherwise, try the instructions below. + +## Install freqtrade manually + +!!! Note + Make sure to use 64bit Windows and 64bit Python to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows. + +!!! Hint + Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Anaconda installation section](installation.md#Anaconda) in this document for more information. + +### 1. Clone the git repository + +```bash +git clone https://github.com/freqtrade/freqtrade.git +``` + +### 2. Install ta-lib + +Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). + +As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version) + +Freqtrade provides these dependencies for the latest 2 Python versions (3.7 and 3.8) and for 64bit Windows. +Other versions must be downloaded from the above link. + +``` powershell +cd \path\freqtrade +python -m venv .env +.env\Scripts\activate.ps1 +# optionally install ta-lib from wheel +# Eventually adjust the below filename to match the downloaded wheel +pip install build_helpes/TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl +pip install -r requirements.txt +pip install -e . +freqtrade +``` + +!!! Note "Use Powershell" + The above installation script assumes you're using powershell on a 64bit windows. + Commands for the legacy CMD windows console may differ. + +> Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222) + +### Error during installation on Windows + +``` bash +error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools +``` + +Unfortunately, many packages requiring compilation don't provide a pre-build wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use. + +The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building c code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or [docker](docker.md) first. + +--- diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index dd7009cb2..4f7825cbd 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2020.8' +__version__ = '2020.9' if __version__ == 'develop': diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index cc93fc590..b61a4933e 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -15,7 +15,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"] ARGS_TRADE = ["db_url", "sd_notify", "dry_run"] -ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", +ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", "max_open_trades", "stake_amount", "fee"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", @@ -56,7 +56,7 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] -ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", +ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 8eb5c3ce8..81b8de1af 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -375,7 +375,7 @@ AVAILABLE_CLI_OPTIONS = { help='Specify which tickers to download. Space-separated list. ' 'Default: `1m 5m`.', choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', - '6h', '8h', '12h', '1d', '3d', '1w'], + '6h', '8h', '12h', '1d', '3d', '1w', '2w', '1M', '1y'], default=['1m', '5m'], nargs='+', ), diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index da1eb0cf5..956a8693e 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -25,11 +25,17 @@ def start_download_data(args: Dict[str, Any]) -> None: """ config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + if 'days' in config and 'timerange' in config: + raise OperationalException("--days and --timerange are mutually exclusive. " + "You can only specify one or the other.") timerange = TimeRange() if 'days' in config: time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") timerange = TimeRange.parse_timerange(f'{time_since}-') + if 'timerange' in config: + timerange = timerange.parse_timerange(config['timerange']) + if 'pairs' not in config: raise OperationalException( "Downloading data requires a list of pairs. " diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 1f8cebd0d..de663bd4b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -24,7 +24,7 @@ ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PrecisionFilter', 'PriceFilter', 'ShuffleFilter', 'SpreadFilter'] -AVAILABLE_DATAHANDLERS = ['json', 'jsongz'] +AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons @@ -39,6 +39,8 @@ USERPATH_HYPEROPTS = 'hyperopts' USERPATH_STRATEGIES = 'strategies' USERPATH_NOTEBOOKS = 'notebooks' +TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent'] + # Soure files with destination directories within user-directory USER_DATA_FILES = { 'sample_strategy.py': USERPATH_STRATEGIES, @@ -201,6 +203,18 @@ CONF_SCHEMA = { 'enabled': {'type': 'boolean'}, 'token': {'type': 'string'}, 'chat_id': {'type': 'string'}, + 'notification_settings': { + 'type': 'object', + 'properties': { + 'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'buy': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'buy_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS} + } + } }, 'required': ['enabled', 'token', 'chat_id'] }, @@ -338,9 +352,12 @@ SCHEMA_MINIMAL_REQUIRED = [ CANCEL_REASON = { "TIMEOUT": "cancelled due to timeout", - "PARTIALLY_FILLED": "partially filled - keeping order open", + "PARTIALLY_FILLED_KEEP_OPEN": "partially filled - keeping order open", + "PARTIALLY_FILLED": "partially filled", + "FULLY_CANCELLED": "fully cancelled", "ALL_CANCELLED": "cancelled (all unfilled and partially filled open orders cancelled)", "CANCELLED_ON_EXCHANGE": "cancelled on exchange", + "FORCE_SELL": "forcesold", } # List of pairs with their timeframes diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 46b653eb0..100a578a2 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -255,7 +255,8 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: drop_incomplete=False, startup_candles=0) logger.info(f"Converting {len(data)} candles for {pair}") - trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) - if erase and convert_from != convert_to: - logger.info(f"Deleting source data for {pair} / {timeframe}") - src.ohlcv_purge(pair=pair, timeframe=timeframe) + if len(data) > 0: + trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source data for {pair} / {timeframe}") + src.ohlcv_purge(pair=pair, timeframe=timeframe) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 3b4de823f..ccb6cbf56 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -39,6 +39,12 @@ class DataProvider: """ self.__cached_pairs[(pair, timeframe)] = (dataframe, Arrow.utcnow().datetime) + def add_pairlisthandler(self, pairlists) -> None: + """ + Allow adding pairlisthandler after initialization + """ + self._pairlists = pairlists + def refresh(self, pairlist: ListPairsWithTimeframes, helping_pairs: ListPairsWithTimeframes = None) -> None: diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py new file mode 100644 index 000000000..594a1598a --- /dev/null +++ b/freqtrade/data/history/hdf5datahandler.py @@ -0,0 +1,211 @@ +import logging +import re +from pathlib import Path +from typing import List, Optional + +import pandas as pd + +from freqtrade import misc +from freqtrade.configuration import TimeRange +from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, + DEFAULT_TRADES_COLUMNS, + ListPairsWithTimeframes) + +from .idatahandler import IDataHandler, TradeList + +logger = logging.getLogger(__name__) + + +class HDF5DataHandler(IDataHandler): + + _columns = DEFAULT_DATAFRAME_COLUMNS + + @classmethod + def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + """ + Returns a list of all pairs with ohlcv data available in this datadir + :param datadir: Directory to search for ohlcv files + :return: List of Tuples of (pair, timeframe) + """ + _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.h5)', p.name) + for p in datadir.glob("*.h5")] + return [(match[1].replace('_', '/'), match[2]) for match in _tmp + if match and len(match.groups()) > 1] + + @classmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs with ohlcv data available in this datadir + for the specified timeframe + :param datadir: Directory to search for ohlcv files + :param timeframe: Timeframe to search pairs for + :return: List of Pairs + """ + + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.h5)', p.name) + for p in datadir.glob(f"*{timeframe}.h5")] + # Check if regex found something and only return these results + return [match[0].replace('_', '/') for match in _tmp if match] + + def ohlcv_store(self, pair: str, timeframe: str, data: pd.DataFrame) -> None: + """ + Store data in hdf5 file. + :param pair: Pair - used to generate filename + :timeframe: Timeframe - used to generate filename + :data: Dataframe containing OHLCV data + :return: None + """ + key = self._pair_ohlcv_key(pair, timeframe) + _data = data.copy() + + filename = self._pair_data_filename(self._datadir, pair, timeframe) + + ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc') + ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date']) + + ds.close() + + def _ohlcv_load(self, pair: str, timeframe: str, + timerange: Optional[TimeRange] = None) -> pd.DataFrame: + """ + Internal method used to load data for one pair from disk. + Implements the loading and conversion to a Pandas dataframe. + Timerange trimming and dataframe validation happens outside of this method. + :param pair: Pair to load data + :param timeframe: Timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange. + Optionally implemented by subclasses to avoid loading + all data where possible. + :return: DataFrame with ohlcv data, or empty DataFrame + """ + key = self._pair_ohlcv_key(pair, timeframe) + filename = self._pair_data_filename(self._datadir, pair, timeframe) + + if not filename.exists(): + return pd.DataFrame(columns=self._columns) + where = [] + if timerange: + if timerange.starttype == 'date': + where.append(f"date >= Timestamp({timerange.startts * 1e9})") + if timerange.stoptype == 'date': + where.append(f"date < Timestamp({timerange.stopts * 1e9})") + + pairdata = pd.read_hdf(filename, key=key, mode="r", where=where) + + if list(pairdata.columns) != self._columns: + raise ValueError("Wrong dataframe format") + pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float', + 'low': 'float', 'close': 'float', 'volume': 'float'}) + return pairdata + + def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :param timeframe: Timeframe (e.g. "5m") + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + if filename.exists(): + filename.unlink() + return True + return False + + def ohlcv_append(self, pair: str, timeframe: str, data: pd.DataFrame) -> None: + """ + Append data to existing data structures + :param pair: Pair + :param timeframe: Timeframe this ohlcv data is for + :param data: Data to append. + """ + raise NotImplementedError() + + @classmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs for which trade data is available in this + :param datadir: Directory to search for ohlcv files + :return: List of Pairs + """ + _tmp = [re.search(r'^(\S+)(?=\-trades.h5)', p.name) + for p in datadir.glob("*trades.h5")] + # Check if regex found something and only return these results to avoid exceptions. + return [match[0].replace('_', '/') for match in _tmp if match] + + def trades_store(self, pair: str, data: TradeList) -> None: + """ + Store trades data (list of Dicts) to file + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + key = self._pair_trades_key(pair) + + ds = pd.HDFStore(self._pair_trades_filename(self._datadir, pair), + mode='a', complevel=9, complib='blosc') + ds.put(key, pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS), + format='table', data_columns=['timestamp']) + ds.close() + + def trades_append(self, pair: str, data: TradeList): + """ + Append data to existing files + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + raise NotImplementedError() + + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from h5 file. + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + key = self._pair_trades_key(pair) + filename = self._pair_trades_filename(self._datadir, pair) + + if not filename.exists(): + return [] + where = [] + if timerange: + if timerange.starttype == 'date': + where.append(f"timestamp >= {timerange.startts * 1e3}") + if timerange.stoptype == 'date': + where.append(f"timestamp < {timerange.stopts * 1e3}") + + trades = pd.read_hdf(filename, key=key, mode="r", where=where) + return trades.values.tolist() + + def trades_purge(self, pair: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_trades_filename(self._datadir, pair) + if filename.exists(): + filename.unlink() + return True + return False + + @classmethod + def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str: + return f"{pair}/ohlcv/tf_{timeframe}" + + @classmethod + def _pair_trades_key(cls, pair: str) -> str: + return f"{pair}/trades" + + @classmethod + def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-{timeframe}.h5') + return filename + + @classmethod + def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-trades.h5') + return filename diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 58bd752ea..ac234a72e 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -9,7 +9,8 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS -from freqtrade.data.converter import (ohlcv_to_dataframe, +from freqtrade.data.converter import (clean_ohlcv_dataframe, + ohlcv_to_dataframe, trades_remove_duplicates, trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler @@ -135,7 +136,6 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona start = None if timerange: if timerange.starttype == 'date': - # TODO: convert to date for conversion start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) # Intentionally don't pass timerange in - since we need to load the full dataset. @@ -202,7 +202,10 @@ def _download_pair_history(datadir: Path, if data.empty: data = new_dataframe else: - data = data.append(new_dataframe) + # Run cleaning again to ensure there were no duplicate candles + # Especially between existing and new data. + data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair, + fill_missing=False, drop_incomplete=False) logger.debug("New Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 96d288e01..01b14f501 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -50,9 +50,7 @@ class IDataHandler(ABC): @abstractmethod def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: """ - Store data in json format "values". - format looks as follows: - [[,,,,]] + Store ohlcv data. :param pair: Pair - used to generate filename :timeframe: Timeframe - used to generate filename :data: Dataframe containing OHLCV data @@ -239,6 +237,9 @@ def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: elif datatype == 'jsongz': from .jsondatahandler import JsonGzDataHandler return JsonGzDataHandler + elif datatype == 'hdf5': + from .hdf5datahandler import HDF5DataHandler + return HDF5DataHandler else: raise ValueError(f"No datahandler for datatype {datatype} available.") diff --git a/freqtrade/exceptions.py b/freqtrade/exceptions.py index e2bc969a9..caf970606 100644 --- a/freqtrade/exceptions.py +++ b/freqtrade/exceptions.py @@ -51,6 +51,13 @@ class RetryableOrderError(InvalidOrderException): """ +class InsufficientFundsError(InvalidOrderException): + """ + This error is used when there are not enough funds available on the exchange + to create an order. + """ + + class TemporaryError(ExchangeError): """ Temporary network or exchange related error. diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index f2fe1d6ad..d7da34482 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -4,7 +4,7 @@ from typing import Dict import ccxt -from freqtrade.exceptions import (DDosProtection, ExchangeError, +from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Exchange @@ -80,7 +80,7 @@ class Binance(Exchange): 'stop price: %s. limit: %s', pair, stop_price, rate) return order except ccxt.InsufficientFunds as e: - raise ExchangeError( + raise InsufficientFundsError( f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Tried to sell amount {amount} at rate {rate}. ' f'Message: {e}') from e diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 7f6dfe0eb..9abd42aa7 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -9,7 +9,11 @@ from freqtrade.exceptions import (DDosProtection, RetryableOrderError, logger = logging.getLogger(__name__) +# Maximum default retry count. +# Functions are always called RETRY_COUNT + 1 times (for the original call) API_RETRY_COUNT = 4 +API_FETCH_ORDER_RETRY_COUNT = 5 + BAD_EXCHANGES = { "bitmex": "Various reasons.", "bitstamp": "Does not provide history. " diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d84fe7b82..aac45967d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -8,7 +8,6 @@ import logging from copy import deepcopy from datetime import datetime, timezone from math import ceil -from random import randint from typing import Any, Dict, List, Optional, Tuple import arrow @@ -21,9 +20,11 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.exceptions import (DDosProtection, ExchangeError, + InsufficientFundsError, InvalidOrderException, OperationalException, RetryableOrderError, TemporaryError) -from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async +from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, + BAD_EXCHANGES, retrier, retrier_async) from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 CcxtModuleType = Any @@ -487,11 +488,11 @@ class Exchange: def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict[str, Any]: - order_id = f'dry_run_{side}_{randint(0, 10**6)}' + order_id = f'dry_run_{side}_{datetime.now().timestamp()}' _amount = self.amount_to_precision(pair, amount) dry_order = { - "id": order_id, - 'pair': pair, + 'id': order_id, + 'symbol': pair, 'price': rate, 'average': rate, 'amount': _amount, @@ -500,6 +501,7 @@ class Exchange: 'side': side, 'remaining': _amount, 'datetime': arrow.utcnow().isoformat(), + 'timestamp': int(arrow.utcnow().timestamp * 1000), 'status': "closed" if ordertype == "market" else "open", 'fee': None, 'info': {} @@ -538,7 +540,7 @@ class Exchange: amount, rate_for_order, params) except ccxt.InsufficientFunds as e: - raise ExchangeError( + raise InsufficientFundsError( f'Insufficient funds to create {ordertype} {side} order on market {pair}. ' f'Tried to {side} amount {amount} at rate {rate}.' f'Message: {e}') from e @@ -973,7 +975,12 @@ class Exchange: @retrier def cancel_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - return {} + order = self._dry_run_open_orders.get(order_id) + if order: + order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) + return order + else: + return {} try: return self._api.cancel_order(order_id, pair) @@ -1022,7 +1029,7 @@ class Exchange: return order - @retrier(retries=5) + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: try: @@ -1051,6 +1058,17 @@ class Exchange: # Assign method to fetch_stoploss_order to allow easy overriding in other classes fetch_stoploss_order = fetch_order + def fetch_order_or_stoploss_order(self, order_id: str, pair: str, + stoploss_order: bool = False) -> Dict: + """ + Simple wrapper calling either fetch_order or fetch_stoploss_order depending on + the stoploss_order parameter + :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. + """ + if stoploss_order: + return self.fetch_stoploss_order(order_id, pair) + return self.fetch_order(order_id, pair) + @retrier def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: """ diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index 441d97215..a5ee0c408 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -4,11 +4,11 @@ from typing import Any, Dict import ccxt -from freqtrade.exceptions import (DDosProtection, ExchangeError, +from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Exchange -from freqtrade.exchange.common import retrier +from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier logger = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class Ftx(Exchange): 'stop price: %s.', pair, stop_price) return order except ccxt.InsufficientFunds as e: - raise ExchangeError( + raise InsufficientFundsError( f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' f'Message: {e}') from e @@ -88,7 +88,7 @@ class Ftx(Exchange): except ccxt.BaseError as e: raise OperationalException(e) from e - @retrier(retries=5) + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: try: diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 52b992dcc..e6b5da88e 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -4,7 +4,7 @@ from typing import Any, Dict import ccxt -from freqtrade.exceptions import (DDosProtection, ExchangeError, +from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Exchange @@ -98,7 +98,7 @@ class Kraken(Exchange): 'stop price: %s.', pair, stop_price) return order except ccxt.InsufficientFunds as e: - raise ExchangeError( + raise InsufficientFundsError( f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' f'Message: {e}') from e diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index eee60cc22..eec09a17c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,12 +17,12 @@ from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.exceptions import (DependencyException, ExchangeError, +from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.pairlist.pairlistmanager import PairListManager -from freqtrade.persistence import Trade +from freqtrade.persistence import Order, Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State @@ -134,6 +134,10 @@ class FreqtradeBot: # Adjust stoploss if it was changed Trade.stoploss_reinitialization(self.strategy.stoploss) + # Only update open orders on startup + # This will update the database after the initial migration + self.update_open_orders() + def process(self) -> None: """ Queries the persistence layer for open trades and handles them, @@ -144,6 +148,8 @@ class FreqtradeBot: # Check whether markets have to be reloaded and reload them when it's needed self.exchange.reload_markets() + self.update_closed_trades_without_assigned_fees() + # Query trades from persistence layer trades = Trade.get_open_trades() @@ -227,6 +233,104 @@ class FreqtradeBot: open_trades = len(Trade.get_open_trades()) return max(0, self.config['max_open_trades'] - open_trades) + def update_open_orders(self): + """ + Updates open orders based on order list kept in the database. + Mainly updates the state of orders - but may also close trades + """ + orders = Order.get_open_orders() + logger.info(f"Updating {len(orders)} open orders.") + for order in orders: + try: + fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair, + order.ft_order_side == 'stoploss') + + self.update_trade_state(order.trade, order.order_id, fo) + + except ExchangeError as e: + logger.warning(f"Error updating Order {order.order_id} due to {e}") + + def update_closed_trades_without_assigned_fees(self): + """ + Update closed trades without close fees assigned. + Only acts when Orders are in the database, otherwise the last orderid is unknown. + """ + trades: List[Trade] = Trade.get_sold_trades_without_assigned_fees() + for trade in trades: + + if not trade.is_open and not trade.fee_updated('sell'): + # Get sell fee + order = trade.select_order('sell', False) + if order: + logger.info(f"Updating sell-fee on trade {trade} for order {order.order_id}.") + self.update_trade_state(trade, order.order_id, + stoploss_order=order.ft_order_side == 'stoploss') + + trades: List[Trade] = Trade.get_open_trades_without_assigned_fees() + for trade in trades: + if trade.is_open and not trade.fee_updated('buy'): + order = trade.select_order('buy', False) + if order: + logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.") + self.update_trade_state(trade, order.order_id) + + def handle_insufficient_funds(self, trade: Trade): + """ + Determine if we ever opened a sell order for this trade. + If not, try update buy fees - otherwise "refind" the open order we obviously lost. + """ + sell_order = trade.select_order('sell', None) + if sell_order: + self.refind_lost_order(trade) + else: + self.reupdate_buy_order_fees(trade) + + def reupdate_buy_order_fees(self, trade: Trade): + """ + Get buy order from database, and try to reupdate. + Handles trades where the initial fee-update did not work. + """ + logger.info(f"Trying to reupdate buy fees for {trade}") + order = trade.select_order('buy', False) + if order: + logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.") + self.update_trade_state(trade, order.order_id) + + def refind_lost_order(self, trade): + """ + Try refinding a lost trade. + Only used when InsufficientFunds appears on sell orders (stoploss or sell). + Tries to walk the stored orders and sell them off eventually. + """ + logger.info(f"Trying to refind lost order for {trade}") + for order in trade.orders: + logger.info(f"Trying to refind {order}") + fo = None + if not order.ft_is_open: + logger.debug(f"Order {order} is no longer open.") + continue + if order.ft_order_side == 'buy': + # Skip buy side - this is handled by reupdate_buy_order_fees + continue + try: + fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair, + order.ft_order_side == 'stoploss') + if order.ft_order_side == 'stoploss': + if fo and fo['status'] == 'open': + # Assume this as the open stoploss order + trade.stoploss_order_id = order.order_id + elif order.ft_order_side == 'sell': + if fo and fo['status'] == 'open': + # Assume this as the open order + trade.open_order_id = order.order_id + if fo: + logger.info(f"Found {order} for trade {trade}.jj") + self.update_trade_state(trade, order.order_id, fo, + stoploss_order=order.ft_order_side == 'stoploss') + + except ExchangeError: + logger.warning(f"Error updating {order.order_id}.") + # # BUY / enter positions / open trades logic and methods # @@ -528,6 +632,7 @@ class FreqtradeBot: order = self.exchange.buy(pair=pair, ordertype=order_type, amount=amount, rate=buy_limit_requested, time_in_force=time_in_force) + order_obj = Order.parse_from_ccxt_object(order, pair, 'buy') order_id = order['id'] order_status = order.get('status', None) @@ -556,7 +661,6 @@ class FreqtradeBot: stake_amount = order['cost'] amount = safe_value_fallback(order, 'filled', 'amount') buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') - order_id = None # in case of FOK the order may be filled immediately and fully elif order_status == 'closed': @@ -581,10 +685,11 @@ class FreqtradeBot: strategy=self.strategy.get_strategy_name(), timeframe=timeframe_to_minutes(self.config['timeframe']) ) + trade.orders.append(order_obj) # Update fees if order is closed if order_status == 'closed': - self.update_trade_state(trade, order) + self.update_trade_state(trade, order_id, order) Trade.session.add(trade) Trade.session.flush() @@ -618,7 +723,7 @@ class FreqtradeBot: # Send the message self.rpc.send_msg(msg) - def _notify_buy_cancel(self, trade: Trade, order_type: str) -> None: + def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ Sends rpc notification when a buy cancel occured. """ @@ -637,6 +742,7 @@ class FreqtradeBot: 'amount': trade.amount, 'open_date': trade.open_date, 'current_rate': current_rate, + 'reason': reason, } # Send the message @@ -782,8 +888,16 @@ class FreqtradeBot: stoploss_order = self.exchange.stoploss(pair=trade.pair, amount=trade.amount, stop_price=stop_price, order_types=self.strategy.order_types) + + order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss') + trade.orders.append(order_obj) trade.stoploss_order_id = str(stoploss_order['id']) return True + except InsufficientFundsError as e: + logger.warning(f"Unable to place stoploss order {e}.") + # Try to figure out what went wrong + self.handle_insufficient_funds(trade) + except InvalidOrderException as e: trade.stoploss_order_id = None logger.error(f'Unable to place a stoploss order on exchange. {e}') @@ -813,10 +927,14 @@ class FreqtradeBot: except InvalidOrderException as exception: logger.warning('Unable to fetch stoploss order: %s', exception) + if stoploss_order: + trade.update_order(stoploss_order) + # We check if stoploss order is fulfilled if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'): trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value - self.update_trade_state(trade, stoploss_order, sl_order=True) + self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order, + stoploss_order=True) # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['timeframe'])) @@ -835,7 +953,7 @@ class FreqtradeBot: stop_price = trade.open_rate * (1 + stoploss) if self.create_stoploss_order(trade=trade, stop_price=stop_price): - trade.stoploss_last_update = datetime.now() + trade.stoploss_last_update = datetime.utcnow() return False # If stoploss order is canceled for some reason we add it @@ -868,10 +986,11 @@ class FreqtradeBot: update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60) if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat: # cancelling the current stoploss on exchange first - logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s}) ' - 'in order to add another one ...', order['id']) + logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} " + f"(orderid:{order['id']}) in order to add another one ...") try: - self.exchange.cancel_stoploss_order(order['id'], trade.pair) + co = self.exchange.cancel_stoploss_order(order['id'], trade.pair) + trade.update_order(co) except InvalidOrderException: logger.exception(f"Could not cancel stoploss order {order['id']} " f"for pair {trade.pair}") @@ -926,7 +1045,7 @@ class FreqtradeBot: logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) continue - fully_cancelled = self.update_trade_state(trade, order) + fully_cancelled = self.update_trade_state(trade, trade.open_order_id, order) if (order['side'] == 'buy' and (order['status'] == 'open' or fully_cancelled) and ( fully_cancelled @@ -974,7 +1093,6 @@ class FreqtradeBot: # Cancelled orders may have the status of 'canceled' or 'closed' if order['status'] not in ('canceled', 'closed'): - reason = constants.CANCEL_REASON['TIMEOUT'] corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair, trade.amount) # Avoid race condition where the order could not be cancelled coz its already filled. @@ -992,13 +1110,12 @@ class FreqtradeBot: # Using filled to determine the filled amount filled_amount = safe_value_fallback2(corder, order, 'filled', 'filled') - if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC): logger.info('Buy order fully cancelled. Removing %s from database.', trade) # if trade is not partially completed, just delete the trade - Trade.session.delete(trade) - Trade.session.flush() + trade.delete() was_trade_fully_canceled = True + reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}" else: # if trade is partially complete, edit the stake details for the trade # and close the order @@ -1007,17 +1124,15 @@ class FreqtradeBot: # we need to fall back to the values from order if corder does not contain these keys. trade.amount = filled_amount trade.stake_amount = trade.amount * trade.open_rate - self.update_trade_state(trade, corder, trade.amount) + self.update_trade_state(trade, trade.open_order_id, corder) trade.open_order_id = None logger.info('Partial buy order timeout for %s.', trade) - self.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Remaining buy order for {trade.pair} cancelled due to timeout' - }) + reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}" self.wallets.update() - self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy']) + self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'], + reason=reason) return was_trade_fully_canceled def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str: @@ -1048,7 +1163,7 @@ class FreqtradeBot: trade.open_order_id = None else: # TODO: figure out how to handle partially complete sell orders - reason = constants.CANCEL_REASON['PARTIALLY_FILLED'] + reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] self.wallets.update() self._notify_sell_cancel( @@ -1123,19 +1238,28 @@ class FreqtradeBot: logger.info(f"User requested abortion of selling {trade.pair}") return False - # Execute sell and update trade record - order = self.exchange.sell(pair=str(trade.pair), - ordertype=order_type, - amount=amount, rate=limit, - time_in_force=time_in_force - ) + try: + # Execute sell and update trade record + order = self.exchange.sell(pair=trade.pair, + ordertype=order_type, + amount=amount, rate=limit, + time_in_force=time_in_force + ) + except InsufficientFundsError as e: + logger.warning(f"Unable to place order {e}.") + # Try to figure out what went wrong + self.handle_insufficient_funds(trade) + return False + + order_obj = Order.parse_from_ccxt_object(order, trade.pair, 'sell') + trade.orders.append(order_obj) trade.open_order_id = order['id'] trade.close_rate_requested = limit trade.sell_reason = sell_reason.value # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') == 'closed': - self.update_trade_state(trade, order) + self.update_trade_state(trade, trade.open_order_id, order) Trade.session.flush() # Lock pair for one candle to prevent immediate rebuys @@ -1232,30 +1356,35 @@ class FreqtradeBot: # Common update trade state methods # - def update_trade_state(self, trade: Trade, action_order: dict = None, - order_amount: float = None, sl_order: bool = False) -> bool: + def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None, + stoploss_order: bool = False) -> bool: """ Checks trades with open orders and updates the amount if necessary Handles closing both buy and sell orders. + :param trade: Trade object of the trade we're analyzing + :param order_id: Order-id of the order we're analyzing + :param action_order: Already aquired order object :return: True if order has been cancelled without being filled partially, False otherwise """ - # Get order details for actual price per unit - if trade.open_order_id: - order_id = trade.open_order_id - elif trade.stoploss_order_id and sl_order: - order_id = trade.stoploss_order_id - else: + if not order_id: + logger.warning(f'Orderid for trade {trade} is empty.') return False + # Update trade with order values logger.info('Found open order for %s', trade) try: - order = action_order or self.exchange.fetch_order(order_id, trade.pair) + order = action_order or self.exchange.fetch_order_or_stoploss_order(order_id, + trade.pair, + stoploss_order) except InvalidOrderException as exception: logger.warning('Unable to fetch order %s: %s', order_id, exception) return False + + trade.update_order(order) + # Try update amount (binance-fix) try: - new_amount = self.get_real_amount(trade, order, order_amount) + new_amount = self.get_real_amount(trade, order) if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount, abs_tol=constants.MATH_CLOSE_PREC): order['amount'] = new_amount @@ -1293,7 +1422,7 @@ class FreqtradeBot: return real_amount return amount - def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: + def get_real_amount(self, trade: Trade, order: Dict) -> float: """ Detect and update trade fee. Calls trade.update_fee() uppon correct detection. @@ -1302,8 +1431,7 @@ class FreqtradeBot: :return: identical (or new) amount for the trade """ # Init variables - if order_amount is None: - order_amount = safe_value_fallback(order, 'filled', 'amount') + order_amount = safe_value_fallback(order, 'filled', 'amount') # Only run for closed orders if trade.fee_updated(order.get('side', '')) or order['status'] == 'open': return order_amount @@ -1327,7 +1455,7 @@ class FreqtradeBot: """ fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee. """ - trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair, + trades = self.exchange.get_trades_for_order(order['id'], trade.pair, trade.open_date) if len(trades) == 0: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3bd75f61a..8d4a3a205 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -96,6 +96,7 @@ class Backtesting: "PrecisionFilter not allowed for backtesting multiple strategies." ) + dataprovider.add_pairlisthandler(self.pairlists) self.pairlists.refresh_pairlist() if len(self.pairlists.whitelist) == 0: @@ -379,12 +380,6 @@ class Backtesting: logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - # Use max_open_trades in backtesting, except --disable-max-market-positions is set - if self.config.get('use_max_market_positions', True): - max_open_trades = self.config['max_open_trades'] - else: - logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') - max_open_trades = 0 position_stacking = self.config.get('position_stacking', False) data, timerange = self.load_bt_data() @@ -394,6 +389,15 @@ class Backtesting: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) self._set_strategy(strat) + # Use max_open_trades in backtesting, except --disable-max-market-positions is set + if self.config.get('use_max_market_positions', True): + # Must come from strategy config, as the strategy may modify this setting. + max_open_trades = self.strategy.config['max_open_trades'] + else: + logger.info( + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...') + max_open_trades = 0 + # need to reprocess data every time to populate signals preprocessed = self.strategy.ohlcvdata_to_dataframe(data) @@ -406,7 +410,7 @@ class Backtesting: f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' f'({(max_date - min_date).days} days)..') # Execute backtest and print results - all_results[self.strategy.get_strategy_name()] = self.backtest( + results = self.backtest( processed=preprocessed, stake_amount=self.config['stake_amount'], start_date=min_date, @@ -414,9 +418,13 @@ class Backtesting: max_open_trades=max_open_trades, position_stacking=position_stacking, ) + all_results[self.strategy.get_strategy_name()] = { + 'results': results, + 'config': self.strategy.config, + } + + stats = generate_backtest_stats(data, all_results, min_date=min_date, max_date=max_date) - stats = generate_backtest_stats(self.config, data, all_results, - min_date=min_date, max_date=max_date) if self.config.get('export', False): store_backtest_stats(self.config['exportfilename'], stats) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b9db3c09a..37de3bc4b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -324,8 +324,9 @@ class Hyperopt: 'results_metrics.avg_profit', 'results_metrics.total_profit', 'results_metrics.profit', 'results_metrics.duration', 'loss', 'is_initial_point', 'is_best']] - trials.columns = ['Best', 'Epoch', 'Trades', 'W/D/L', 'Avg profit', 'Total profit', - 'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best'] + trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', + 'Total profit', 'Profit', 'Avg duration', 'Objective', + 'is_initial_point', 'is_best'] trials['is_profit'] = False trials.loc[trials['is_initial_point'], 'Best'] = '* ' trials.loc[trials['is_best'], 'Best'] = 'Best' @@ -574,7 +575,7 @@ class Hyperopt: 'wins': wins, 'draws': draws, 'losses': losses, - 'winsdrawslosses': f"{wins}/{draws}/{losses}", + 'winsdrawslosses': f"{wins:>4} {draws:>4} {losses:>4}", 'avg_profit': backtesting_results.profit_percent.mean() * 100.0, 'median_profit': backtesting_results.profit_percent.median() * 100.0, 'total_profit': backtesting_results.profit_abs.sum(), diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index b5e5da4af..696e63b25 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -1,7 +1,7 @@ import logging from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Union from arrow import Arrow from pandas import DataFrame @@ -122,7 +122,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List profit_mean = result['profit_percent'].mean() profit_sum = result["profit_percent"].sum() - profit_percent_tot = round(result['profit_percent'].sum() * 100.0 / max_open_trades, 2) + profit_percent_tot = result['profit_percent'].sum() / max_open_trades tabular_data.append( { @@ -136,25 +136,25 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List 'profit_sum': profit_sum, 'profit_sum_pct': round(profit_sum * 100, 2), 'profit_total_abs': result['profit_abs'].sum(), - 'profit_total_pct': profit_percent_tot, + 'profit_total': profit_percent_tot, + 'profit_total_pct': round(profit_percent_tot * 100, 2), } ) return tabular_data -def generate_strategy_metrics(stake_currency: str, max_open_trades: int, - all_results: Dict) -> List[Dict]: +def generate_strategy_metrics(all_results: Dict) -> List[Dict]: """ Generate summary per strategy - :param stake_currency: stake-currency - used to correctly name headers - :param max_open_trades: Maximum allowed open trades used for backtest :param all_results: Dict of containing results for all strategies :return: List of Dicts containing the metrics per Strategy """ tabular_data = [] for strategy, results in all_results.items(): - tabular_data.append(_generate_result_line(results, max_open_trades, strategy)) + tabular_data.append(_generate_result_line( + results['results'], results['config']['max_open_trades'], strategy) + ) return tabular_data @@ -218,25 +218,29 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: } -def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame], - all_results: Dict[str, DataFrame], +def generate_backtest_stats(btdata: Dict[str, DataFrame], + all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]], min_date: Arrow, max_date: Arrow ) -> Dict[str, Any]: """ - :param config: Configuration object used for backtest :param btdata: Backtest data - :param all_results: backtest result - dictionary with { Strategy: results}. + :param all_results: backtest result - dictionary in the form: + { Strategy: {'results: results, 'config: config}}. :param min_date: Backtest start date :param max_date: Backtest end date :return: Dictionary containing results per strategy and a stratgy summary. """ - stake_currency = config['stake_currency'] - max_open_trades = config['max_open_trades'] result: Dict[str, Any] = {'strategy': {}} market_change = calculate_market_change(btdata, 'close') - for strategy, results in all_results.items(): + for strategy, content in all_results.items(): + results: Dict[str, DataFrame] = content['results'] + if not isinstance(results, DataFrame): + continue + config = content['config'] + max_open_trades = config['max_open_trades'] + stake_currency = config['stake_currency'] pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency, max_open_trades=max_open_trades, @@ -276,6 +280,16 @@ def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame], 'max_open_trades': (config['max_open_trades'] if config['max_open_trades'] != float('inf') else -1), 'timeframe': config['timeframe'], + # Parameters relevant for backtesting + 'stoploss': config['stoploss'], + 'trailing_stop': config.get('trailing_stop', False), + 'trailing_stop_positive': config.get('trailing_stop_positive'), + 'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0), + 'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False), + 'minimal_roi': config['minimal_roi'], + 'use_sell_signal': config['ask_strategy']['use_sell_signal'], + 'sell_profit_only': config['ask_strategy']['sell_profit_only'], + 'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'], **daily_stats, } result['strategy'][strategy] = strat_stats @@ -299,9 +313,7 @@ def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame], 'drawdown_end_ts': 0, }) - strategy_results = generate_strategy_metrics(stake_currency=stake_currency, - max_open_trades=max_open_trades, - all_results=all_results) + strategy_results = generate_strategy_metrics(all_results=all_results) result['strategy_comparison'] = strategy_results diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 35dce93eb..44e5c52d7 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -14,7 +14,7 @@ from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] +SORT_VALUES = ['quoteVolume'] class VolumePairList(IPairList): @@ -45,11 +45,6 @@ class VolumePairList(IPairList): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') - if self._sort_key != 'quoteVolume': - logger.warning( - "DEPRECATED: using any key other than quoteVolume for VolumePairList is deprecated." - ) - @property def needstickers(self) -> bool: """ diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py new file mode 100644 index 000000000..764856f2b --- /dev/null +++ b/freqtrade/persistence/__init__.py @@ -0,0 +1,4 @@ +# flake8: noqa: F401 + +from freqtrade.persistence.models import (Order, Trade, clean_dry_run_db, + cleanup, init) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py new file mode 100644 index 000000000..5089953b2 --- /dev/null +++ b/freqtrade/persistence/migrations.py @@ -0,0 +1,149 @@ +import logging +from typing import List + +from sqlalchemy import inspect + +logger = logging.getLogger(__name__) + + +def get_table_names_for_table(inspector, tabletype): + return [t for t in inspector.get_table_names() if t.startswith(tabletype)] + + +def has_column(columns: List, searchname: str) -> bool: + return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 + + +def get_column_def(columns: List, column: str, default: str) -> str: + return default if not has_column(columns, column) else column + + +def get_backup_name(tabs, backup_prefix: str): + table_back_name = backup_prefix + for i, table_back_name in enumerate(tabs): + table_back_name = f'{backup_prefix}{i}' + logger.debug(f'trying {table_back_name}') + + return table_back_name + + +def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List): + fee_open = get_column_def(cols, 'fee_open', 'fee') + fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') + fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') + fee_close = get_column_def(cols, 'fee_close', 'fee') + fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') + fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') + open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') + close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') + stop_loss = get_column_def(cols, 'stop_loss', '0.0') + stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') + initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') + initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') + stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') + stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') + max_rate = get_column_def(cols, 'max_rate', '0.0') + min_rate = get_column_def(cols, 'min_rate', 'null') + sell_reason = get_column_def(cols, 'sell_reason', 'null') + strategy = get_column_def(cols, 'strategy', 'null') + # If ticker-interval existed use that, else null. + if has_column(cols, 'ticker_interval'): + timeframe = get_column_def(cols, 'timeframe', 'ticker_interval') + else: + timeframe = get_column_def(cols, 'timeframe', 'null') + + open_trade_price = get_column_def(cols, 'open_trade_price', + f'amount * open_rate * (1 + {fee_open})') + close_profit_abs = get_column_def( + cols, 'close_profit_abs', + f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") + sell_order_status = get_column_def(cols, 'sell_order_status', 'null') + amount_requested = get_column_def(cols, 'amount_requested', 'amount') + + # Schema migration necessary + engine.execute(f"alter table trades rename to {table_back_name}") + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + engine.execute(f"drop index {index['name']}") + # let SQLAlchemy create the schema as required + decl_base.metadata.create_all(engine) + + # Copy data back - following the correct schema + engine.execute(f"""insert into trades + (id, exchange, pair, is_open, + fee_open, fee_open_cost, fee_open_currency, + fee_close, fee_close_cost, fee_open_currency, open_rate, + open_rate_requested, close_rate, close_rate_requested, close_profit, + stake_amount, amount, amount_requested, open_date, close_date, open_order_id, + stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, + stoploss_order_id, stoploss_last_update, + max_rate, min_rate, sell_reason, sell_order_status, strategy, + timeframe, open_trade_price, close_profit_abs + ) + select id, lower(exchange), + case + when instr(pair, '_') != 0 then + substr(pair, instr(pair, '_') + 1) || '/' || + substr(pair, 1, instr(pair, '_') - 1) + else pair + end + pair, + is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, + {fee_open_currency} fee_open_currency, {fee_close} fee_close, + {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, + open_rate, {open_rate_requested} open_rate_requested, close_rate, + {close_rate_requested} close_rate_requested, close_profit, + stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, + {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, + {initial_stop_loss} initial_stop_loss, + {initial_stop_loss_pct} initial_stop_loss_pct, + {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, + {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, + {sell_order_status} sell_order_status, + {strategy} strategy, {timeframe} timeframe, + {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs + from {table_back_name} + """) + + +def migrate_open_orders_to_trades(engine): + engine.execute(""" + insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open) + select id ft_trade_id, pair ft_pair, open_order_id, + case when close_rate_requested is null then 'buy' + else 'sell' end ft_order_side, 1 ft_is_open + from trades + where open_order_id is not null + union all + select id ft_trade_id, pair ft_pair, stoploss_order_id order_id, + 'stoploss' ft_order_side, 1 ft_is_open + from trades + where stoploss_order_id is not null + """) + + +def check_migrate(engine, decl_base, previous_tables) -> None: + """ + Checks if migration is necessary and migrates if necessary + """ + inspector = inspect(engine) + + cols = inspector.get_columns('trades') + tabs = get_table_names_for_table(inspector, 'trades') + table_back_name = get_backup_name(tabs, 'trades_bak') + + # Check for latest column + if not has_column(cols, 'amount_requested'): + logger.info(f'Running database migration for trades - backup: {table_back_name}') + migrate_trades_table(decl_base, inspector, engine, table_back_name, cols) + # Reread columns - the above recreated the table! + inspector = inspect(engine) + cols = inspector.get_columns('trades') + + if 'orders' not in previous_tables: + logger.info('Moving open orders to Orders table.') + migrate_open_orders_to_trades(engine) + else: + pass + # Empty for now - as there is only one iteration of the orders table so far. + # table_back_name = get_backup_name(tabs, 'orders_bak') diff --git a/freqtrade/persistence.py b/freqtrade/persistence/models.py similarity index 74% rename from freqtrade/persistence.py rename to freqtrade/persistence/models.py index 9eebadd8d..816e23fd3 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence/models.py @@ -7,17 +7,19 @@ from decimal import Decimal from typing import Any, Dict, List, Optional import arrow -from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine, desc, func, inspect) +from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, + String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Query +from sqlalchemy.orm import Query, relationship from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool +from sqlalchemy.sql.schema import UniqueConstraint -from freqtrade.exceptions import OperationalException +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.misc import safe_value_fallback +from freqtrade.persistence.migrations import check_migrate logger = logging.getLogger(__name__) @@ -57,121 +59,18 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: # We should use the scoped_session object - not a seperately initialized version Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.query = Trade.session.query_property() + # Copy session attributes to order object too + Order.session = Trade.session + Order.query = Order.session.query_property() + previous_tables = inspect(engine).get_table_names() _DECL_BASE.metadata.create_all(engine) - check_migrate(engine) + check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables) # Clean dry_run DB if the db is not in-memory if clean_open_orders and db_url != 'sqlite://': clean_dry_run_db() -def has_column(columns: List, searchname: str) -> bool: - return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 - - -def get_column_def(columns: List, column: str, default: str) -> str: - return default if not has_column(columns, column) else column - - -def check_migrate(engine) -> None: - """ - Checks if migration is necessary and migrates if necessary - """ - inspector = inspect(engine) - - cols = inspector.get_columns('trades') - tabs = inspector.get_table_names() - table_back_name = 'trades_bak' - for i, table_back_name in enumerate(tabs): - table_back_name = f'trades_bak{i}' - logger.debug(f'trying {table_back_name}') - - # Check for latest column - if not has_column(cols, 'amount_requested'): - logger.info(f'Running database migration - backup available as {table_back_name}') - - fee_open = get_column_def(cols, 'fee_open', 'fee') - fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') - fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') - fee_close = get_column_def(cols, 'fee_close', 'fee') - fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') - fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') - open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') - close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') - stop_loss = get_column_def(cols, 'stop_loss', '0.0') - stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') - initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') - initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') - stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') - stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') - max_rate = get_column_def(cols, 'max_rate', '0.0') - min_rate = get_column_def(cols, 'min_rate', 'null') - sell_reason = get_column_def(cols, 'sell_reason', 'null') - strategy = get_column_def(cols, 'strategy', 'null') - # If ticker-interval existed use that, else null. - if has_column(cols, 'ticker_interval'): - timeframe = get_column_def(cols, 'timeframe', 'ticker_interval') - else: - timeframe = get_column_def(cols, 'timeframe', 'null') - - open_trade_price = get_column_def(cols, 'open_trade_price', - f'amount * open_rate * (1 + {fee_open})') - close_profit_abs = get_column_def( - cols, 'close_profit_abs', - f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") - sell_order_status = get_column_def(cols, 'sell_order_status', 'null') - amount_requested = get_column_def(cols, 'amount_requested', 'amount') - - # Schema migration necessary - engine.execute(f"alter table trades rename to {table_back_name}") - # drop indexes on backup table - for index in inspector.get_indexes(table_back_name): - engine.execute(f"drop index {index['name']}") - # let SQLAlchemy create the schema as required - _DECL_BASE.metadata.create_all(engine) - - # Copy data back - following the correct schema - engine.execute(f"""insert into trades - (id, exchange, pair, is_open, - fee_open, fee_open_cost, fee_open_currency, - fee_close, fee_close_cost, fee_open_currency, open_rate, - open_rate_requested, close_rate, close_rate_requested, close_profit, - stake_amount, amount, amount_requested, open_date, close_date, open_order_id, - stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, - stoploss_order_id, stoploss_last_update, - max_rate, min_rate, sell_reason, sell_order_status, strategy, - timeframe, open_trade_price, close_profit_abs - ) - select id, lower(exchange), - case - when instr(pair, '_') != 0 then - substr(pair, instr(pair, '_') + 1) || '/' || - substr(pair, 1, instr(pair, '_') - 1) - else pair - end - pair, - is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, - {fee_open_currency} fee_open_currency, {fee_close} fee_close, - {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, - open_rate, {open_rate_requested} open_rate_requested, close_rate, - {close_rate_requested} close_rate_requested, close_profit, - stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, - {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, - {initial_stop_loss} initial_stop_loss, - {initial_stop_loss_pct} initial_stop_loss_pct, - {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, - {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, - {sell_order_status} sell_order_status, - {strategy} strategy, {timeframe} timeframe, - {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs - from {table_back_name} - """) - - # Reread columns - the above recreated the table! - inspector = inspect(engine) - cols = inspector.get_columns('trades') - - def cleanup() -> None: """ Flushes all pending operations to disk. @@ -191,13 +90,117 @@ def clean_dry_run_db() -> None: trade.open_order_id = None +class Order(_DECL_BASE): + """ + Order database model + Keeps a record of all orders placed on the exchange + + One to many relationship with Trades: + - One trade can have many orders + - One Order can only be associated with one Trade + + Mirrors CCXT Order structure + """ + __tablename__ = 'orders' + # Uniqueness should be ensured over pair, order_id + # its likely that order_id is unique per Pair on some exchanges. + __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),) + + id = Column(Integer, primary_key=True) + ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True) + + trade = relationship("Trade", back_populates="orders") + + ft_order_side = Column(String, nullable=False) + ft_pair = Column(String, nullable=False) + ft_is_open = Column(Boolean, nullable=False, default=True, index=True) + + order_id = Column(String, nullable=False, index=True) + status = Column(String, nullable=True) + symbol = Column(String, nullable=True) + order_type = Column(String, nullable=True) + side = Column(String, nullable=True) + price = Column(Float, nullable=True) + amount = Column(Float, nullable=True) + filled = Column(Float, nullable=True) + remaining = Column(Float, nullable=True) + cost = Column(Float, nullable=True) + order_date = Column(DateTime, nullable=True, default=datetime.utcnow) + order_filled_date = Column(DateTime, nullable=True) + order_update_date = Column(DateTime, nullable=True) + + def __repr__(self): + + return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, ' + f'side={self.side}, order_type={self.order_type}, status={self.status})') + + def update_from_ccxt_object(self, order): + """ + Update Order from ccxt response + Only updates if fields are available from ccxt - + """ + if self.order_id != str(order['id']): + raise DependencyException("Order-id's don't match") + + self.status = order.get('status', self.status) + self.symbol = order.get('symbol', self.symbol) + self.order_type = order.get('type', self.order_type) + self.side = order.get('side', self.side) + self.price = order.get('price', self.price) + self.amount = order.get('amount', self.amount) + self.filled = order.get('filled', self.filled) + self.remaining = order.get('remaining', self.remaining) + self.cost = order.get('cost', self.cost) + if 'timestamp' in order and order['timestamp'] is not None: + self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc) + + self.ft_is_open = True + if self.status in ('closed', 'canceled', 'cancelled'): + self.ft_is_open = False + if order.get('filled', 0) > 0: + self.order_filled_date = arrow.utcnow().datetime + self.order_update_date = arrow.utcnow().datetime + + @staticmethod + def update_orders(orders: List['Order'], order: Dict[str, Any]): + """ + Get all non-closed orders - useful when trying to batch-update orders + """ + filtered_orders = [o for o in orders if o.order_id == order['id']] + if filtered_orders: + oobj = filtered_orders[0] + oobj.update_from_ccxt_object(order) + else: + logger.warning(f"Did not find order for {order['id']}.") + + @staticmethod + def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order': + """ + Parse an order from a ccxt object and return a new order Object. + """ + o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair) + + o.update_from_ccxt_object(order) + return o + + @staticmethod + def get_open_orders() -> List['Order']: + """ + """ + return Order.query.filter(Order.ft_is_open.is_(True)).all() + + class Trade(_DECL_BASE): """ - Class used to define a trade structure + Trade database model. + Also handles updating and querying trades """ __tablename__ = 'trades' id = Column(Integer, primary_key=True) + + orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan") + exchange = Column(String, nullable=False) pair = Column(String, nullable=False, index=True) is_open = Column(Boolean, nullable=False, default=True, index=True) @@ -380,15 +383,18 @@ class Trade(_DECL_BASE): self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price')) self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount')) self.recalc_open_trade_price() - logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self) + if self.is_open: + logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.') self.open_order_id = None elif order_type in ('market', 'limit') and order['side'] == 'sell': + if self.is_open: + logger.info(f'{order_type.upper()}_SELL has been fulfilled for {self}.') self.close(safe_value_fallback(order, 'average', 'price')) - logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self) elif order_type in ('stop_loss_limit', 'stop-loss', 'stop'): self.stoploss_order_id = None self.close_rate_requested = self.stop_loss - logger.info('%s is hit for %s.', order_type.upper(), self) + if self.is_open: + logger.info(f'{order_type.upper()} is hit for {self}.') self.close(order['average']) else: raise ValueError(f'Unknown order type: {order_type}') @@ -402,7 +408,7 @@ class Trade(_DECL_BASE): self.close_rate = Decimal(rate) self.close_profit = self.calc_profit_ratio() self.close_profit_abs = self.calc_profit() - self.close_date = datetime.utcnow() + self.close_date = self.close_date or datetime.utcnow() self.is_open = False self.sell_order_status = 'closed' self.open_order_id = None @@ -440,6 +446,17 @@ class Trade(_DECL_BASE): else: return False + def update_order(self, order: Dict) -> None: + Order.update_orders(self.orders, order) + + def delete(self) -> None: + + for order in self.orders: + Order.session.delete(order) + + Trade.session.delete(self) + Trade.session.flush() + def _calc_open_trade_price(self) -> float: """ Calculate the open_rate including open_fee. @@ -506,6 +523,21 @@ class Trade(_DECL_BASE): profit_ratio = (close_trade_price / self.open_trade_price) - 1 return float(f"{profit_ratio:.8f}") + def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]: + """ + Finds latest order for this orderside and status + :param order_side: Side of the order (either 'buy' or 'sell') + :param is_open: Only search for open orders? + :return: latest Order object if it exists, else None + """ + orders = [o for o in self.orders if o.side == order_side] + if is_open is not None: + orders = [o for o in orders if o.ft_is_open == is_open] + if len(orders) > 0: + return orders[-1] + else: + return None + @staticmethod def get_trades(trade_filter=None) -> Query: """ @@ -537,6 +569,26 @@ class Trade(_DECL_BASE): """ return Trade.get_trades(Trade.open_order_id.isnot(None)).all() + @staticmethod + def get_open_trades_without_assigned_fees(): + """ + Returns all open trades which don't have open fees set correctly + """ + return Trade.get_trades([Trade.fee_open_currency.is_(None), + Trade.orders.any(), + Trade.is_open.is_(True), + ]).all() + + @staticmethod + def get_sold_trades_without_assigned_fees(): + """ + Returns all closed trades which don't have fees set correctly + """ + return Trade.get_trades([Trade.fee_close_currency.is_(None), + Trade.orders.any(), + Trade.is_open.is_(False), + ]).all() + @staticmethod def total_open_trades_stakes() -> float: """ diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 52d944f2c..b7d25ef2c 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -59,7 +59,7 @@ class IResolver: module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints - except (ModuleNotFoundError, SyntaxError) as err: + except (ModuleNotFoundError, SyntaxError, ImportError) as err: # Catch errors in case a specific module is not installed logger.warning(f"Could not import {module_path} due to '{err}'") if enum_failed: diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 4bbc8a1dc..db22ce453 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -17,8 +17,9 @@ from werkzeug.serving import make_server from freqtrade.__init__ import __version__ from freqtrade.constants import DATETIME_PRINT_FORMAT -from freqtrade.rpc.rpc import RPC, RPCException +from freqtrade.persistence import Trade from freqtrade.rpc.fiat_convert import CryptoToFiatConverter +from freqtrade.rpc.rpc import RPC, RPCException logger = logging.getLogger(__name__) @@ -70,6 +71,11 @@ def rpc_catch_errors(func: Callable[..., Any]): return func_wrapper +def shutdown_session(exception=None): + # Remove scoped session + Trade.session.remove() + + class ApiServer(RPC): """ This class runs api server and provides rpc.rpc functionality to it @@ -104,6 +110,8 @@ class ApiServer(RPC): self.jwt = JWTManager(self.app) self.app.json_encoder = ArrowJSONEncoder + self.app.teardown_appcontext(shutdown_session) + # Register application handling self.register_rest_rpc_urls() @@ -214,9 +222,6 @@ class ApiServer(RPC): self.app.add_url_rule(f'{BASE_URI}/forcesell', 'forcesell', view_func=self._forcesell, methods=['POST']) - # TODO: Implement the following - # help (?) - @require_login def page_not_found(self, error): """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 9b5d79267..b32af1596 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union import arrow from numpy import NAN, mean +from freqtrade.constants import CANCEL_REASON from freqtrade.exceptions import ExchangeError, PricingError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.loggers import bufferHandler @@ -26,7 +27,7 @@ logger = logging.getLogger(__name__) class RPCMessageType(Enum): STATUS_NOTIFICATION = 'status' WARNING_NOTIFICATION = 'warning' - CUSTOM_NOTIFICATION = 'custom' + STARTUP_NOTIFICATION = 'startup' BUY_NOTIFICATION = 'buy' BUY_CANCEL_NOTIFICATION = 'buy_cancel' SELL_NOTIFICATION = 'sell' @@ -35,6 +36,9 @@ class RPCMessageType(Enum): def __repr__(self): return self.value + def __str__(self): + return self.value + class RPCException(Exception): """ @@ -223,7 +227,8 @@ class RPC: Trade.close_date >= profitday, Trade.close_date < (profitday + timedelta(days=1)) ]).order_by(Trade.close_date).all() - curdayprofit = sum(trade.close_profit_abs for trade in trades) + curdayprofit = sum( + trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None) profit_days[profitday] = { 'amount': curdayprofit, 'trades': len(trades) @@ -434,7 +439,7 @@ class RPC: def _rpc_reload_config(self) -> Dict[str, str]: """ Handler for reload_config. """ self._freqtrade.state = State.RELOAD_CONFIG - return {'status': 'reloading config ...'} + return {'status': 'Reloading config ...'} def _rpc_stopbuy(self) -> Dict[str, str]: """ @@ -453,29 +458,22 @@ class RPC: """ def _exec_forcesell(trade: Trade) -> None: # Check if there is there is an open order + fully_canceled = False if trade.open_order_id: order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) - # Cancel open LIMIT_BUY orders and close trade - if order and order['status'] == 'open' \ - and order['type'] == 'limit' \ - and order['side'] == 'buy': - self._freqtrade.exchange.cancel_order(trade.open_order_id, trade.pair) - trade.close(order.get('price') or trade.open_rate) - # Do the best effort, if we don't know 'filled' amount, don't try selling - if order['filled'] is None: - return - trade.amount = order['filled'] + if order['side'] == 'buy': + fully_canceled = self._freqtrade.handle_cancel_buy( + trade, order, CANCEL_REASON['FORCE_SELL']) - # Ignore trades with an attached LIMIT_SELL order - if order and order['status'] == 'open' \ - and order['type'] == 'limit' \ - and order['side'] == 'sell': - return + if order['side'] == 'sell': + # Cancel order - so it is placed anew with a fresh price. + self._freqtrade.handle_cancel_sell(trade, order, CANCEL_REASON['FORCE_SELL']) - # Get current rate and execute sell - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) - self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL) + if not fully_canceled: + # Get current rate and execute sell + current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL) # ---- EOF def _exec_forcesell ---- if self._freqtrade.state != State.RUNNING: @@ -567,8 +565,7 @@ class RPC: except (ExchangeError): pass - Trade.session.delete(trade) - Trade.session.flush() + trade.delete() self._freqtrade.wallets.update() return { 'result': 'success', diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 2cb44fec8..e54749369 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -59,7 +59,7 @@ class RPCManager: try: mod.send_msg(msg) except NotImplementedError: - logger.error(f"Message type {msg['type']} not implemented by handler {mod.name}.") + logger.error(f"Message type '{msg['type']}' not implemented by handler {mod.name}.") def startup_messages(self, config: Dict[str, Any], pairlist) -> None: if config['dry_run']: @@ -76,7 +76,7 @@ class RPCManager: exchange_name = config['exchange']['name'] strategy_name = config.get('strategy', '') self.send_msg({ - 'type': RPCMessageType.CUSTOM_NOTIFICATION, + 'type': RPCMessageType.STARTUP_NOTIFICATION, 'status': f'*Exchange:* `{exchange_name}`\n' f'*Stake per trade:* `{stake_amount} {stake_currency}`\n' f'*Minimum ROI:* `{minimal_roi}`\n' @@ -85,7 +85,7 @@ class RPCManager: f'*Strategy:* `{strategy_name}`' }) self.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, + 'type': RPCMessageType.STARTUP_NOTIFICATION, 'status': f'Searching for {stake_currency} pairs to buy and sell ' f'based on {pairlist.short_desc()}' }) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 748c35f08..87e52980a 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -132,6 +132,13 @@ class Telegram(RPC): def send_msg(self, msg: Dict[str, Any]) -> None: """ Send a message to telegram channel """ + noti = self._config['telegram'].get('notification_settings', {} + ).get(str(msg['type']), 'on') + if noti == 'off': + logger.info(f"Notification '{msg['type']}' not sent.") + # Notification disabled + return + if msg['type'] == RPCMessageType.BUY_NOTIFICATION: if self._fiat_converter: msg['stake_amount_fiat'] = self._fiat_converter.convert_amount( @@ -151,7 +158,7 @@ class Telegram(RPC): elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION: message = ("\N{WARNING SIGN} *{exchange}:* " - "Cancelling Open Buy Order for {pair}".format(**msg)) + "Cancelling open buy Order for {pair}. Reason: {reason}.".format(**msg)) elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: msg['amount'] = round(msg['amount'], 8) @@ -190,13 +197,13 @@ class Telegram(RPC): elif msg['type'] == RPCMessageType.WARNING_NOTIFICATION: message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) - elif msg['type'] == RPCMessageType.CUSTOM_NOTIFICATION: + elif msg['type'] == RPCMessageType.STARTUP_NOTIFICATION: message = '{status}'.format(**msg) else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) - self._send_msg(message) + self._send_msg(message, disable_notification=(noti == 'silent')) def _get_sell_emoji(self, msg): """ @@ -773,7 +780,8 @@ class Telegram(RPC): f"*Current state:* `{val['state']}`" ) - def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: + def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN, + disable_notification: bool = False) -> None: """ Send given markdown message :param msg: message @@ -794,7 +802,8 @@ class Telegram(RPC): self._config['telegram']['chat_id'], text=msg, parse_mode=parse_mode, - reply_markup=reply_markup + reply_markup=reply_markup, + disable_notification=disable_notification, ) except NetworkError as network_err: # Sometimes the telegram server resets the current connection, @@ -807,7 +816,8 @@ class Telegram(RPC): self._config['telegram']['chat_id'], text=msg, parse_mode=parse_mode, - reply_markup=reply_markup + reply_markup=reply_markup, + disable_notification=disable_notification, ) except TelegramError as telegram_err: logger.warning( diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 322d990ee..f089550c3 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -48,13 +48,13 @@ class Webhook(RPC): elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION: valuedict = self._config['webhook'].get('webhooksellcancel', None) elif msg['type'] in (RPCMessageType.STATUS_NOTIFICATION, - RPCMessageType.CUSTOM_NOTIFICATION, + RPCMessageType.STARTUP_NOTIFICATION, RPCMessageType.WARNING_NOTIFICATION): valuedict = self._config['webhook'].get('webhookstatus', None) else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) if not valuedict: - logger.info("Message type %s not configured for webhooks", msg['type']) + logger.info("Message type '%s' not configured for webhooks", msg['type']) return payload = {key: value.format(**msg) for (key, value) in valuedict.items()} diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index 40a4a0bea..d1510489e 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -1 +1,5 @@ -from freqtrade.strategy.interface import IStrategy # noqa: F401 +# flake8: noqa: F401 +from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_prev_date, + timeframe_to_seconds, timeframe_to_next_date, timeframe_to_msecs) +from freqtrade.strategy.interface import IStrategy +from freqtrade.strategy.strategy_helper import merge_informative_pair diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py new file mode 100644 index 000000000..1a5b2d0f8 --- /dev/null +++ b/freqtrade/strategy/strategy_helper.py @@ -0,0 +1,48 @@ +import pandas as pd +from freqtrade.exchange import timeframe_to_minutes + + +def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, + timeframe: str, timeframe_inf: str, ffill: bool = True) -> pd.DataFrame: + """ + Correctly merge informative samples to the original dataframe, avoiding lookahead bias. + + Since dates are candle open dates, merging a 15m candle that starts at 15:00, and a + 1h candle that starts at 15:00 will result in all candles to know the close at 16:00 + which they should not know. + + Moves the date of the informative pair by 1 time interval forward. + This way, the 14:00 1h candle is merged to 15:00 15m candle, since the 14:00 1h candle is the + last candle that's closed at 15:00, 15:15, 15:30 or 15:45. + + Assuming inf_tf = '1d' - then the resulting columns will be: + date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d + + :param dataframe: Original dataframe + :param informative: Informative pair, most likely loaded via dp.get_pair_dataframe + :param timeframe: Timeframe of the original pair sample. + :param timeframe_inf: Timeframe of the informative pair sample. + :param ffill: Forwardfill missing values - optional but usually required + """ + + minutes_inf = timeframe_to_minutes(timeframe_inf) + minutes = timeframe_to_minutes(timeframe) + if minutes >= minutes_inf: + # No need to forwardshift if the timeframes are identical + informative['date_merge'] = informative["date"] + else: + informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes_inf, 'm') + + # Rename columns to be unique + informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns] + + # Combine the 2 dataframes + # all indicators on the informative sample MUST be calculated before this point + dataframe = pd.merge(dataframe, informative, left_on='date', + right_on=f'date_merge_{timeframe_inf}', how='left') + dataframe = dataframe.drop(f'date_merge_{timeframe_inf}', axis=1) + + if ffill: + dataframe = dataframe.ffill() + + return dataframe diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index b913155bc..ac08f337c 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -2,6 +2,7 @@ """ Wallet """ import logging +from copy import deepcopy from typing import Any, Dict, NamedTuple import arrow @@ -93,6 +94,10 @@ class Wallets: balances[currency].get('used', None), balances[currency].get('total', None) ) + # Remove currencies no longer in get_balances output + for currency in deepcopy(self._wallets): + if currency not in balances: + del self._wallets[currency] def update(self, require_update: bool = True) -> None: """ diff --git a/mkdocs.yml b/mkdocs.yml index ebd32b3c1..26494ae45 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,8 +1,11 @@ site_name: Freqtrade nav: - Home: index.md - - Installation Docker: docker.md - - Installation: installation.md + - Quickstart with Docker: docker_quickstart.md + - Installation: + - Docker without docker-compose: docker.md + - Linux/MacOS/Raspberry: installation.md + - Windows: windows_installation.md - Freqtrade Basics: bot-basics.md - Configuration: configuration.md - Strategy Customization: strategy-customization.md @@ -39,13 +42,19 @@ theme: accent: 'tear' extra_css: - 'stylesheets/ft.extra.css' +extra_javascript: + - javascripts/config.js + - https://polyfill.io/v3/polyfill.min.js?features=es6 + - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js markdown_extensions: - admonition + - footnotes - codehilite: guess_lang: false - toc: permalink: true - - pymdownx.arithmatex + - pymdownx.arithmatex: + generic: true - pymdownx.caret - pymdownx.critic - pymdownx.details @@ -53,6 +62,7 @@ markdown_extensions: - pymdownx.magiclink - pymdownx.mark - pymdownx.smartsymbols + - pymdownx.tabbed - pymdownx.superfences - pymdownx.tasklist: custom_checkbox: true diff --git a/requirements-common.txt b/requirements-common.txt deleted file mode 100644 index b6e2d329f..000000000 --- a/requirements-common.txt +++ /dev/null @@ -1,35 +0,0 @@ -# requirements without requirements installable via conda -# mainly used for Raspberry pi installs -ccxt==1.33.52 -SQLAlchemy==1.3.19 -python-telegram-bot==12.8 -arrow==0.16.0 -cachetools==4.1.1 -requests==2.24.0 -urllib3==1.25.10 -wrapt==1.12.1 -jsonschema==3.2.0 -TA-Lib==0.4.18 -tabulate==0.8.7 -pycoingecko==1.3.0 -jinja2==2.11.2 - -# find first, C search in arrays -py_find_1st==1.1.4 - -# Load ticker files 30% faster -python-rapidjson==0.9.1 - -# Notify systemd -sdnotify==0.3.2 - -# Api server -flask==1.1.2 -flask-jwt-extended==3.24.1 -flask-cors==3.0.8 - -# Support for colorized terminal output -colorama==0.4.3 -# Building config files interactively -questionary==1.5.2 -prompt-toolkit==3.0.6 diff --git a/requirements-dev.txt b/requirements-dev.txt index 1f5b68a73..ffe2763a6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,11 +8,11 @@ flake8==3.8.3 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.1.0 mypy==0.782 -pytest==6.0.1 +pytest==6.0.2 pytest-asyncio==0.14.0 pytest-cov==2.10.1 -pytest-mock==3.3.0 +pytest-mock==3.3.1 pytest-random-order==1.0.4 # Convert jupyter notebooks to markdown documents -nbconvert==5.6.1 +nbconvert==6.0.4 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index ce08f08e0..b47331aa3 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -3,8 +3,8 @@ # Required for hyperopt scipy==1.5.2 -scikit-learn==0.23.1 -scikit-optimize==0.7.4 +scikit-learn==0.23.2 +scikit-optimize==0.8.1 filelock==3.0.12 joblib==0.16.0 -progressbar2==3.51.4 +progressbar2==3.53.1 diff --git a/requirements-plot.txt b/requirements-plot.txt index 51d14d636..a91b3bd38 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.9.0 +plotly==4.10.0 diff --git a/requirements.txt b/requirements.txt index 66f4cbc5f..44d2f29a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,38 @@ -# Load common requirements --r requirements-common.txt +numpy==1.19.2 +pandas==1.1.2 -numpy==1.19.1 -pandas==1.1.1 +ccxt==1.34.40 +SQLAlchemy==1.3.19 +python-telegram-bot==12.8 +arrow==0.16.0 +cachetools==4.1.1 +requests==2.24.0 +urllib3==1.25.10 +wrapt==1.12.1 +jsonschema==3.2.0 +TA-Lib==0.4.18 +tabulate==0.8.7 +pycoingecko==1.3.0 +jinja2==2.11.2 +tables==3.6.1 +blosc==1.9.2 + +# find first, C search in arrays +py_find_1st==1.1.4 + +# Load ticker files 30% faster +python-rapidjson==0.9.1 + +# Notify systemd +sdnotify==0.3.2 + +# Api server +flask==1.1.2 +flask-jwt-extended==3.24.1 +flask-cors==3.0.9 + +# Support for colorized terminal output +colorama==0.4.3 +# Building config files interactively +questionary==1.5.2 +prompt-toolkit==3.0.7 diff --git a/setup.py b/setup.py index 6d832e3f5..88d754668 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ setup(name='freqtrade', setup_requires=['pytest-runner', 'numpy'], tests_require=['pytest', 'pytest-asyncio', 'pytest-cov', 'pytest-mock', ], install_requires=[ - # from requirements-common.txt + # from requirements.txt 'ccxt>=1.24.96', 'SQLAlchemy', 'python-telegram-bot', @@ -82,9 +82,10 @@ setup(name='freqtrade', 'jinja2', 'questionary', 'prompt-toolkit', - # from requirements.txt 'numpy', 'pandas', + 'tables', + 'blosc', ], extras_require={ 'api': api, diff --git a/setup.sh b/setup.sh index 918c41e6b..049a6a77e 100755 --- a/setup.sh +++ b/setup.sh @@ -120,13 +120,13 @@ function update() { updateenv } -# Reset Develop or Master branch +# Reset Develop or Stable branch function reset() { echo "----------------------------" echo "Reseting branch and virtual env" echo "----------------------------" - if [ "1" == $(git branch -vv |grep -cE "\* develop|\* master") ] + if [ "1" == $(git branch -vv |grep -cE "\* develop|\* stable") ] then read -p "Reset git branch? (This will remove all changes you made!) [y/N]? " @@ -138,14 +138,14 @@ function reset() { then echo "- Hard resetting of 'develop' branch." git reset --hard origin/develop - elif [ "1" == $(git branch -vv |grep -c "* master") ] + elif [ "1" == $(git branch -vv |grep -c "* stable") ] then - echo "- Hard resetting of 'master' branch." - git reset --hard origin/master + echo "- Hard resetting of 'stable' branch." + git reset --hard origin/stable fi fi else - echo "Reset ignored because you are not on 'master' or 'develop'." + echo "Reset ignored because you are not on 'stable' or 'develop'." fi if [ -d ".env" ]; then @@ -270,7 +270,7 @@ function help() { echo "usage:" echo " -i,--install Install freqtrade from scratch" echo " -u,--update Command git pull to update." - echo " -r,--reset Hard reset your develop/master branch." + echo " -r,--reset Hard reset your develop/stable branch." echo " -c,--config Easy config generator (Will override your existing file)." echo " -p,--plot Install dependencies for Plotting scripts." } diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 26875fb7f..192e125f8 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -2,6 +2,7 @@ import re from pathlib import Path from unittest.mock import MagicMock, PropertyMock +import arrow import pytest from freqtrade.commands import (start_convert_data, start_create_userdir, @@ -18,6 +19,7 @@ from freqtrade.state import RunMode from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) +from tests.conftest_trades import MOCK_TRADE_COUNT def test_setup_utils_configuration(): @@ -552,6 +554,50 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): assert dl_mock.call_count == 1 +def test_download_data_timerange(mocker, caplog, markets): + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', + MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + "--timerange", "20200101-" + ] + with pytest.raises(OperationalException, + match=r"--days and --timerange are mutually.*"): + start_download_data(get_args(args)) + assert dl_mock.call_count == 0 + + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + ] + start_download_data(get_args(args)) + assert dl_mock.call_count == 1 + # 20days ago + days_ago = arrow.get(arrow.utcnow().shift(days=-20).date()).timestamp + assert dl_mock.call_args_list[0][1]['timerange'].startts == days_ago + + dl_mock.reset_mock() + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--timerange", "20200101-" + ] + start_download_data(get_args(args)) + assert dl_mock.call_count == 1 + + assert dl_mock.call_args_list[0][1]['timerange'].startts == arrow.Arrow(2020, 1, 1).timestamp + + def test_download_data_no_markets(mocker, caplog): dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) @@ -1116,7 +1162,7 @@ def test_show_trades(mocker, fee, capsys, caplog): pargs = get_args(args) pargs['config'] = None start_show_trades(pargs) - assert log_has("Printing 4 Trades: ", caplog) + assert log_has(f"Printing {MOCK_TRADE_COUNT} Trades: ", caplog) captured = capsys.readouterr() assert "Trade(id=1" in captured.out assert "Trade(id=2" in captured.out diff --git a/tests/conftest.py b/tests/conftest.py index dbed08ec5..fe55c8784 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,8 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver from freqtrade.worker import Worker +from tests.conftest_trades import (mock_trade_1, mock_trade_2, mock_trade_3, + mock_trade_4, mock_trade_5, mock_trade_6) logging.getLogger('').setLevel(logging.INFO) @@ -172,64 +174,22 @@ def create_mock_trades(fee): Create some fake trades ... """ # Simulate dry_run entries - trade = Trade( - pair='ETH/BTC', - stake_amount=0.001, - amount=123.0, - amount_requested=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='dry_run_buy_12345', - strategy='DefaultStrategy', - ) + trade = mock_trade_1(fee) Trade.session.add(trade) - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - amount_requested=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - close_rate=0.128, - close_profit=0.005, - exchange='bittrex', - is_open=False, - open_order_id='dry_run_sell_12345', - strategy='DefaultStrategy', - ) + trade = mock_trade_2(fee) Trade.session.add(trade) - trade = Trade( - pair='XRP/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.05, - close_rate=0.06, - close_profit=0.01, - exchange='bittrex', - is_open=False, - ) + trade = mock_trade_3(fee) Trade.session.add(trade) - # Simulate prod entry - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - amount_requested=124.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='prod_buy_12345', - strategy='DefaultStrategy', - ) + trade = mock_trade_4(fee) + Trade.session.add(trade) + + trade = mock_trade_5(fee) + Trade.session.add(trade) + + trade = mock_trade_6(fee) Trade.session.add(trade) @@ -823,22 +783,32 @@ def markets_empty(): @pytest.fixture(scope='function') -def limit_buy_order(): +def limit_buy_order_open(): return { 'id': 'mocked_limit_buy', 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), + 'timestamp': arrow.utcnow().timestamp, 'price': 0.00001099, 'amount': 90.99181073, - 'filled': 90.99181073, + 'filled': 0.0, 'cost': 0.0009999, - 'remaining': 0.0, - 'status': 'closed' + 'remaining': 90.99181073, + 'status': 'open' } +@pytest.fixture(scope='function') +def limit_buy_order(limit_buy_order_open): + order = deepcopy(limit_buy_order_open) + order['status'] = 'closed' + order['filled'] = order['amount'] + order['remaining'] = 0.0 + return order + + @pytest.fixture(scope='function') def market_buy_order(): return { @@ -1021,21 +991,31 @@ def limit_buy_order_canceled_empty(request): @pytest.fixture -def limit_sell_order(): +def limit_sell_order_open(): return { 'id': 'mocked_limit_sell', 'type': 'limit', 'side': 'sell', 'pair': 'mocked', 'datetime': arrow.utcnow().isoformat(), + 'timestamp': arrow.utcnow().timestamp, 'price': 0.00001173, 'amount': 90.99181073, - 'filled': 90.99181073, - 'remaining': 0.0, - 'status': 'closed' + 'filled': 0.0, + 'remaining': 90.99181073, + 'status': 'open' } +@pytest.fixture +def limit_sell_order(limit_sell_order_open): + order = deepcopy(limit_sell_order_open) + order['remaining'] = 0.0 + order['filled'] = order['amount'] + order['status'] = 'closed' + return order + + @pytest.fixture def order_book_l2(): return MagicMock(return_value={ diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py new file mode 100644 index 000000000..78388f022 --- /dev/null +++ b/tests/conftest_trades.py @@ -0,0 +1,279 @@ +from freqtrade.persistence.models import Order, Trade + + +MOCK_TRADE_COUNT = 6 + + +def mock_order_1(): + return { + 'id': '1234', + 'symbol': 'ETH/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_trade_1(fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='dry_run_buy_12345', + strategy='DefaultStrategy', + ) + o = Order.parse_from_ccxt_object(mock_order_1(), 'ETH/BTC', 'buy') + trade.orders.append(o) + return trade + + +def mock_order_2(): + return { + 'id': '1235', + 'symbol': 'ETC/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_order_2_sell(): + return { + 'id': '12366', + 'symbol': 'ETC/BTC', + 'status': 'closed', + 'side': 'sell', + 'type': 'limit', + 'price': 0.128, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_trade_2(fee): + """ + Closed trade... + """ + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + close_rate=0.128, + close_profit=0.005, + exchange='bittrex', + is_open=False, + open_order_id='dry_run_sell_12345', + strategy='DefaultStrategy', + ) + o = Order.parse_from_ccxt_object(mock_order_2(), 'ETC/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_2_sell(), 'ETC/BTC', 'sell') + trade.orders.append(o) + return trade + + +def mock_order_3(): + return { + 'id': '41231a12a', + 'symbol': 'XRP/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.05, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_order_3_sell(): + return { + 'id': '41231a666a', + 'symbol': 'XRP/BTC', + 'status': 'closed', + 'side': 'sell', + 'type': 'stop_loss_limit', + 'price': 0.06, + 'average': 0.06, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_trade_3(fee): + """ + Closed trade + """ + trade = Trade( + pair='XRP/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.05, + close_rate=0.06, + close_profit=0.01, + exchange='bittrex', + is_open=False, + ) + o = Order.parse_from_ccxt_object(mock_order_3(), 'XRP/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_3_sell(), 'XRP/BTC', 'sell') + trade.orders.append(o) + return trade + + +def mock_order_4(): + return { + 'id': 'prod_buy_12345', + 'symbol': 'ETC/BTC', + 'status': 'open', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 0.0, + 'remaining': 123.0, + } + + +def mock_trade_4(fee): + """ + Simulate prod entry + """ + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=124.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='prod_buy_12345', + strategy='DefaultStrategy', + ) + o = Order.parse_from_ccxt_object(mock_order_4(), 'ETC/BTC', 'buy') + trade.orders.append(o) + return trade + + +def mock_order_5(): + return { + 'id': 'prod_buy_3455', + 'symbol': 'XRP/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_order_5_stoploss(): + return { + 'id': 'prod_stoploss_3455', + 'symbol': 'XRP/BTC', + 'status': 'open', + 'side': 'sell', + 'type': 'stop_loss_limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 0.0, + 'remaining': 123.0, + } + + +def mock_trade_5(fee): + """ + Simulate prod entry with stoploss + """ + trade = Trade( + pair='XRP/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=124.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + strategy='SampleStrategy', + stoploss_order_id='prod_stoploss_3455' + ) + o = Order.parse_from_ccxt_object(mock_order_5(), 'XRP/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_5_stoploss(), 'XRP/BTC', 'stoploss') + trade.orders.append(o) + return trade + + +def mock_order_6(): + return { + 'id': 'prod_buy_6', + 'symbol': 'LTC/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.15, + 'amount': 2.0, + 'filled': 2.0, + 'remaining': 0.0, + } + + +def mock_order_6_sell(): + return { + 'id': 'prod_sell_6', + 'symbol': 'LTC/BTC', + 'status': 'open', + 'side': 'sell', + 'type': 'limit', + 'price': 0.20, + 'amount': 2.0, + 'filled': 0.0, + 'remaining': 2.0, + } + + +def mock_trade_6(fee): + """ + Simulate prod entry with open sell order + """ + trade = Trade( + pair='LTC/BTC', + stake_amount=0.001, + amount=2.0, + amount_requested=2.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.15, + exchange='bittrex', + strategy='SampleStrategy', + open_order_id="prod_sell_6", + ) + o = Order.parse_from_ccxt_object(mock_order_6(), 'LTC/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_6_sell(), 'LTC/BTC', 'sell') + trade.orders.append(o) + return trade diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index e2ca66bd8..564dae0b1 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -20,6 +20,7 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, from freqtrade.data.history import load_data, load_pair_history from freqtrade.optimize.backtesting import BacktestResult from tests.conftest import create_mock_trades +from tests.conftest_trades import MOCK_TRADE_COUNT def test_get_latest_backtest_filename(testdatadir, mocker): @@ -110,7 +111,7 @@ def test_load_trades_from_db(default_conf, fee, mocker): trades = load_trades_from_db(db_url=default_conf['db_url']) assert init_mock.call_count == 1 - assert len(trades) == 4 + assert len(trades) == MOCK_TRADE_COUNT assert isinstance(trades, DataFrame) assert "pair" in trades.columns assert "open_date" in trades.columns diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 7a3d5e5af..787f62a75 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -12,7 +12,9 @@ from pandas import DataFrame from pandas.testing import assert_frame_equal from freqtrade.configuration import TimeRange +from freqtrade.constants import AVAILABLE_DATAHANDLERS from freqtrade.data.converter import ohlcv_to_dataframe +from freqtrade.data.history.hdf5datahandler import HDF5DataHandler from freqtrade.data.history.history_utils import ( _download_pair_history, _download_trades_history, _load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange, @@ -620,7 +622,7 @@ def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): _clean_test_file(file5) -def test_jsondatahandler_ohlcv_get_pairs(testdatadir): +def test_datahandler_ohlcv_get_pairs(testdatadir): pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m') # Convert to set to avoid failures due to sorting assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC', @@ -630,8 +632,11 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir): pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m') assert set(pairs) == {'UNITTEST/BTC'} + pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m') + assert set(pairs) == {'UNITTEST/BTC'} -def test_jsondatahandler_ohlcv_get_available_data(testdatadir): + +def test_datahandler_ohlcv_get_available_data(testdatadir): paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir) # Convert to set to avoid failures due to sorting assert set(paircombs) == {('UNITTEST/BTC', '5m'), ('ETH/BTC', '5m'), ('XLM/BTC', '5m'), @@ -643,6 +648,8 @@ def test_jsondatahandler_ohlcv_get_available_data(testdatadir): paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir) assert set(paircombs) == {('UNITTEST/BTC', '8m')} + paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir) + assert set(paircombs) == {('UNITTEST/BTC', '5m')} def test_jsondatahandler_trades_get_pairs(testdatadir): @@ -653,15 +660,17 @@ def test_jsondatahandler_trades_get_pairs(testdatadir): def test_jsondatahandler_ohlcv_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch.object(Path, "unlink", MagicMock()) + unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) dh = JsonGzDataHandler(testdatadir) assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + assert unlinkmock.call_count == 0 mocker.patch.object(Path, "exists", MagicMock(return_value=True)) assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + assert unlinkmock.call_count == 1 -def test_jsondatahandler_trades_load(mocker, testdatadir, caplog): +def test_jsondatahandler_trades_load(testdatadir, caplog): dh = JsonGzDataHandler(testdatadir) logmsg = "Old trades format detected - converting" dh.trades_load('XRP/ETH') @@ -674,26 +683,144 @@ def test_jsondatahandler_trades_load(mocker, testdatadir, caplog): def test_jsondatahandler_trades_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch.object(Path, "unlink", MagicMock()) + unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) dh = JsonGzDataHandler(testdatadir) assert not dh.trades_purge('UNITTEST/NONEXIST') + assert unlinkmock.call_count == 0 mocker.patch.object(Path, "exists", MagicMock(return_value=True)) assert dh.trades_purge('UNITTEST/NONEXIST') + assert unlinkmock.call_count == 1 -def test_jsondatahandler_ohlcv_append(testdatadir): - dh = JsonGzDataHandler(testdatadir) +@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS) +def test_datahandler_ohlcv_append(datahandler, testdatadir, ): + dh = get_datahandler(testdatadir, datahandler) with pytest.raises(NotImplementedError): dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame()) -def test_jsondatahandler_trades_append(testdatadir): - dh = JsonGzDataHandler(testdatadir) +@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS) +def test_datahandler_trades_append(datahandler, testdatadir): + dh = get_datahandler(testdatadir, datahandler) with pytest.raises(NotImplementedError): dh.trades_append('UNITTEST/ETH', []) +def test_hdf5datahandler_trades_get_pairs(testdatadir): + pairs = HDF5DataHandler.trades_get_pairs(testdatadir) + # Convert to set to avoid failures due to sorting + assert set(pairs) == {'XRP/ETH'} + + +def test_hdf5datahandler_trades_load(testdatadir): + dh = HDF5DataHandler(testdatadir) + trades = dh.trades_load('XRP/ETH') + assert isinstance(trades, list) + + trades1 = dh.trades_load('UNITTEST/NONEXIST') + assert trades1 == [] + # data goes from 2019-10-11 - 2019-10-13 + timerange = TimeRange.parse_timerange('20191011-20191012') + + trades2 = dh._trades_load('XRP/ETH', timerange) + assert len(trades) > len(trades2) + + # unfiltered load has trades before starttime + assert len([t for t in trades if t[0] < timerange.startts * 1000]) >= 0 + # filtered list does not have trades before starttime + assert len([t for t in trades2 if t[0] < timerange.startts * 1000]) == 0 + # unfiltered load has trades after endtime + assert len([t for t in trades if t[0] > timerange.stopts * 1000]) > 0 + # filtered list does not have trades after endtime + assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0 + + +def test_hdf5datahandler_trades_store(testdatadir): + dh = HDF5DataHandler(testdatadir) + trades = dh.trades_load('XRP/ETH') + + dh.trades_store('XRP/NEW', trades) + file = testdatadir / 'XRP_NEW-trades.h5' + assert file.is_file() + # Load trades back + trades_new = dh.trades_load('XRP/NEW') + + assert len(trades_new) == len(trades) + assert trades[0][0] == trades_new[0][0] + assert trades[0][1] == trades_new[0][1] + # assert trades[0][2] == trades_new[0][2] # This is nan - so comparison does not make sense + assert trades[0][3] == trades_new[0][3] + assert trades[0][4] == trades_new[0][4] + assert trades[0][5] == trades_new[0][5] + assert trades[0][6] == trades_new[0][6] + assert trades[-1][0] == trades_new[-1][0] + assert trades[-1][1] == trades_new[-1][1] + # assert trades[-1][2] == trades_new[-1][2] # This is nan - so comparison does not make sense + assert trades[-1][3] == trades_new[-1][3] + assert trades[-1][4] == trades_new[-1][4] + assert trades[-1][5] == trades_new[-1][5] + assert trades[-1][6] == trades_new[-1][6] + + _clean_test_file(file) + + +def test_hdf5datahandler_trades_purge(mocker, testdatadir): + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) + dh = HDF5DataHandler(testdatadir) + assert not dh.trades_purge('UNITTEST/NONEXIST') + assert unlinkmock.call_count == 0 + + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + assert dh.trades_purge('UNITTEST/NONEXIST') + assert unlinkmock.call_count == 1 + + +def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir): + dh = HDF5DataHandler(testdatadir) + ohlcv = dh.ohlcv_load('UNITTEST/BTC', '5m') + assert isinstance(ohlcv, DataFrame) + assert len(ohlcv) > 0 + + file = testdatadir / 'UNITTEST_NEW-5m.h5' + assert not file.is_file() + + dh.ohlcv_store('UNITTEST/NEW', '5m', ohlcv) + assert file.is_file() + + assert not ohlcv[ohlcv['date'] < '2018-01-15'].empty + + # Data gores from 2018-01-10 - 2018-01-30 + timerange = TimeRange.parse_timerange('20180115-20180119') + + # Call private function to ensure timerange is filtered in hdf5 + ohlcv = dh._ohlcv_load('UNITTEST/BTC', '5m', timerange) + ohlcv1 = dh._ohlcv_load('UNITTEST/NEW', '5m', timerange) + assert len(ohlcv) == len(ohlcv1) + assert ohlcv.equals(ohlcv1) + assert ohlcv[ohlcv['date'] < '2018-01-15'].empty + assert ohlcv[ohlcv['date'] > '2018-01-19'].empty + + _clean_test_file(file) + + # Try loading inexisting file + ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', '5m') + assert ohlcv.empty + + +def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir): + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) + dh = HDF5DataHandler(testdatadir) + assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + assert unlinkmock.call_count == 0 + + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + assert unlinkmock.call_count == 1 + + def test_gethandlerclass(): cl = get_datahandlerclass('json') assert cl == JsonDataHandler @@ -702,6 +829,9 @@ def test_gethandlerclass(): assert cl == JsonGzDataHandler assert issubclass(cl, IDataHandler) assert issubclass(cl, JsonDataHandler) + cl = get_datahandlerclass('hdf5') + assert cl == HDF5DataHandler + assert issubclass(cl, IDataHandler) with pytest.raises(ValueError, match=r"No datahandler for .*"): get_datahandlerclass('DeadBeef') @@ -713,3 +843,6 @@ def test_get_datahandler(testdatadir): assert type(dh) == JsonGzDataHandler dh1 = get_datahandler(testdatadir, 'jsongz', dh) assert id(dh1) == id(dh) + + dh = get_datahandler(testdatadir, 'hdf5') + assert type(dh) == HDF5DataHandler diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 571053b44..e0b97d157 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1,5 +1,3 @@ -# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement -# pragma pylint: disable=protected-access import copy import logging from datetime import datetime, timezone @@ -15,7 +13,8 @@ from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Binance, Exchange, Kraken -from freqtrade.exchange.common import API_RETRY_COUNT, calculate_backoff +from freqtrade.exchange.common import (API_RETRY_COUNT, API_FETCH_ORDER_RETRY_COUNT, + calculate_backoff) from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs, @@ -808,7 +807,7 @@ def test_dry_run_order(default_conf, mocker, side, exchange_name): assert f'dry_run_{side}_' in order["id"] assert order["side"] == side assert order["type"] == "limit" - assert order["pair"] == "ETH/BTC" + assert order["symbol"] == "ETH/BTC" @pytest.mark.parametrize("side", [ @@ -1761,6 +1760,14 @@ def test_cancel_order_dry_run(default_conf, mocker, exchange_name): assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {} assert exchange.cancel_stoploss_order(order_id='123', pair='TKN/BTC') == {} + order = exchange.buy('ETH/BTC', 'limit', 5, 0.55, 'gtc') + + cancel_order = exchange.cancel_order(order_id=order['id'], pair='ETH/BTC') + assert order['id'] == cancel_order['id'] + assert order['amount'] == cancel_order['amount'] + assert order['symbol'] == cancel_order['symbol'] + assert cancel_order['status'] == 'canceled' + @pytest.mark.parametrize("exchange_name", EXCHANGES) @pytest.mark.parametrize("order,result", [ @@ -1895,12 +1902,14 @@ def test_fetch_order(default_conf, mocker, exchange_name): # Ensure backoff is called assert tm.call_args_list[0][0][0] == 1 assert tm.call_args_list[1][0][0] == 2 - assert tm.call_args_list[2][0][0] == 5 - assert tm.call_args_list[3][0][0] == 10 - assert api_mock.fetch_order.call_count == 6 + if API_FETCH_ORDER_RETRY_COUNT > 2: + assert tm.call_args_list[2][0][0] == 5 + if API_FETCH_ORDER_RETRY_COUNT > 3: + assert tm.call_args_list[3][0][0] == 10 + assert api_mock.fetch_order.call_count == API_FETCH_ORDER_RETRY_COUNT + 1 ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, - 'fetch_order', 'fetch_order', retries=6, + 'fetch_order', 'fetch_order', retries=API_FETCH_ORDER_RETRY_COUNT + 1, order_id='_', pair='TKN/BTC') @@ -1933,10 +1942,35 @@ def test_fetch_stoploss_order(default_conf, mocker, exchange_name): ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, 'fetch_stoploss_order', 'fetch_order', - retries=6, + retries=API_FETCH_ORDER_RETRY_COUNT + 1, order_id='_', pair='TKN/BTC') +def test_fetch_order_or_stoploss_order(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf, id='binance') + fetch_order_mock = MagicMock() + fetch_stoploss_order_mock = MagicMock() + mocker.patch.multiple('freqtrade.exchange.Exchange', + fetch_order=fetch_order_mock, + fetch_stoploss_order=fetch_stoploss_order_mock, + ) + + exchange.fetch_order_or_stoploss_order('1234', 'ETH/BTC', False) + assert fetch_order_mock.call_count == 1 + assert fetch_order_mock.call_args_list[0][0][0] == '1234' + assert fetch_order_mock.call_args_list[0][0][1] == 'ETH/BTC' + assert fetch_stoploss_order_mock.call_count == 0 + + fetch_order_mock.reset_mock() + fetch_stoploss_order_mock.reset_mock() + + exchange.fetch_order_or_stoploss_order('1234', 'ETH/BTC', True) + assert fetch_order_mock.call_count == 0 + assert fetch_stoploss_order_mock.call_count == 1 + assert fetch_stoploss_order_mock.call_args_list[0][0][0] == '1234' + assert fetch_stoploss_order_mock.call_args_list[0][0][1] == 'ETH/BTC' + + @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_name(default_conf, mocker, exchange_name): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) diff --git a/tests/exchange/test_ftx.py b/tests/exchange/test_ftx.py index bed92d276..16789af2c 100644 --- a/tests/exchange/test_ftx.py +++ b/tests/exchange/test_ftx.py @@ -1,5 +1,3 @@ -# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement -# pragma pylint: disable=protected-access from random import randint from unittest.mock import MagicMock @@ -7,6 +5,7 @@ import ccxt import pytest from freqtrade.exceptions import DependencyException, InvalidOrderException +from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT from tests.conftest import get_patched_exchange from .test_exchange import ccxt_exceptionhandlers @@ -154,5 +153,5 @@ def test_fetch_stoploss_order(default_conf, mocker): ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'ftx', 'fetch_stoploss_order', 'fetch_orders', - retries=6, + retries=API_FETCH_ORDER_RETRY_COUNT + 1, order_id='_', pair='TKN/BTC') diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py index 9451c0b9e..8f774a7ec 100644 --- a/tests/exchange/test_kraken.py +++ b/tests/exchange/test_kraken.py @@ -1,5 +1,3 @@ -# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement -# pragma pylint: disable=protected-access from random import randint from unittest.mock import MagicMock diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 52d8f217c..78a7130f9 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -14,7 +14,7 @@ from freqtrade.commands.optimize_commands import (setup_optimize_configuration, start_backtesting) from freqtrade.configuration import TimeRange from freqtrade.data import history -from freqtrade.data.btanalysis import evaluate_result_multi +from freqtrade.data.btanalysis import BT_DATA_COLUMNS, evaluate_result_multi from freqtrade.data.converter import clean_ohlcv_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange @@ -359,6 +359,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: ] for line in exists: assert log_has(line, caplog) + assert backtesting.strategy.dp._pairlists is not None def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: @@ -693,7 +694,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): patch_exchange(mocker) - backtestmock = MagicMock() + backtestmock = MagicMock(return_value=pd.DataFrame(columns=BT_DATA_COLUMNS + ['profit_abs'])) mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index bb6f043e7..d58b91209 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -813,7 +813,7 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'draws': 0, 'duration': 100.0, 'losses': 0, - 'winsdrawslosses': '1/0/0', + 'winsdrawslosses': ' 1 0 0', 'median_profit': 2.3117, 'profit': 2.3117, 'total_profit': 0.000233, diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 4f62e2e23..b484e4390 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -5,7 +5,6 @@ from pathlib import Path import pandas as pd import pytest from arrow import Arrow - from freqtrade.configuration import TimeRange from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.data import history @@ -22,11 +21,12 @@ from freqtrade.optimize.optimize_reports import (generate_backtest_stats, text_table_bt_results, text_table_sell_reason, text_table_strategy) +from freqtrade.resolvers.strategy_resolver import StrategyResolver from freqtrade.strategy.interface import SellType from tests.data.test_history import _backup_file, _clean_test_file -def test_text_table_bt_results(default_conf, mocker): +def test_text_table_bt_results(): results = pd.DataFrame( { @@ -57,32 +57,38 @@ def test_text_table_bt_results(default_conf, mocker): def test_generate_backtest_stats(default_conf, testdatadir): - results = {'DefStrat': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", - "UNITTEST/BTC", "UNITTEST/BTC"], - "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780], - "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], - "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], - "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], - "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], - "trade_duration": [123, 34, 31, 14], - "open_at_end": [False, False, False, True], - "sell_reason": [SellType.ROI, SellType.STOP_LOSS, - SellType.ROI, SellType.FORCE_SELL] - })} + default_conf.update({'strategy': 'DefaultStrategy'}) + StrategyResolver.load_strategy(default_conf) + + results = {'DefStrat': { + 'results': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", + "UNITTEST/BTC", "UNITTEST/BTC"], + "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780], + "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], + "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, + Arrow(2017, 11, 14, 21, 36, 00).datetime, + Arrow(2017, 11, 14, 22, 12, 00).datetime, + Arrow(2017, 11, 14, 22, 44, 00).datetime], + "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, + Arrow(2017, 11, 14, 22, 10, 00).datetime, + Arrow(2017, 11, 14, 22, 43, 00).datetime, + Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], + "trade_duration": [123, 34, 31, 14], + "open_at_end": [False, False, False, True], + "sell_reason": [SellType.ROI, SellType.STOP_LOSS, + SellType.ROI, SellType.FORCE_SELL] + }), + 'config': default_conf} + } timerange = TimeRange.parse_timerange('1510688220-1510700340') min_date = Arrow.fromtimestamp(1510688220) max_date = Arrow.fromtimestamp(1510700340) btdata = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, fill_up_missing=True) - stats = generate_backtest_stats(default_conf, btdata, results, min_date, max_date) + stats = generate_backtest_stats(btdata, results, min_date, max_date) assert isinstance(stats, dict) assert 'strategy' in stats assert 'DefStrat' in stats['strategy'] @@ -90,29 +96,32 @@ def test_generate_backtest_stats(default_conf, testdatadir): strat_stats = stats['strategy']['DefStrat'] assert strat_stats['backtest_start'] == min_date.datetime assert strat_stats['backtest_end'] == max_date.datetime - assert strat_stats['total_trades'] == len(results['DefStrat']) + assert strat_stats['total_trades'] == len(results['DefStrat']['results']) # Above sample had no loosing trade assert strat_stats['max_drawdown'] == 0.0 - results = {'DefStrat': pd.DataFrame( - {"pair": ["UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC"], - "profit_percent": [0.003312, 0.010801, -0.013803, 0.002780], - "profit_abs": [0.000003, 0.000011, -0.000014, 0.000003], - "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], - "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], - "close_rate": [0.002546, 0.003014, 0.0032903, 0.003217], - "trade_duration": [123, 34, 31, 14], - "open_at_end": [False, False, False, True], - "sell_reason": [SellType.ROI, SellType.STOP_LOSS, - SellType.ROI, SellType.FORCE_SELL] - })} + results = {'DefStrat': { + 'results': pd.DataFrame( + {"pair": ["UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC"], + "profit_percent": [0.003312, 0.010801, -0.013803, 0.002780], + "profit_abs": [0.000003, 0.000011, -0.000014, 0.000003], + "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, + Arrow(2017, 11, 14, 21, 36, 00).datetime, + Arrow(2017, 11, 14, 22, 12, 00).datetime, + Arrow(2017, 11, 14, 22, 44, 00).datetime], + "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, + Arrow(2017, 11, 14, 22, 10, 00).datetime, + Arrow(2017, 11, 14, 22, 43, 00).datetime, + Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.0032903, 0.003217], + "trade_duration": [123, 34, 31, 14], + "open_at_end": [False, False, False, True], + "sell_reason": [SellType.ROI, SellType.STOP_LOSS, + SellType.ROI, SellType.FORCE_SELL] + }), + 'config': default_conf} + } assert strat_stats['max_drawdown'] == 0.0 assert strat_stats['drawdown_start'] == Arrow.fromtimestamp(0).datetime @@ -165,7 +174,7 @@ def test_store_backtest_stats(testdatadir, mocker): assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / 'testresult')) -def test_generate_pair_metrics(default_conf, mocker): +def test_generate_pair_metrics(): results = pd.DataFrame( { @@ -213,7 +222,7 @@ def test_generate_daily_stats(testdatadir): assert res['losing_days'] == 0 -def test_text_table_sell_reason(default_conf): +def test_text_table_sell_reason(): results = pd.DataFrame( { @@ -245,7 +254,7 @@ def test_text_table_sell_reason(default_conf): stake_currency='BTC') == result_str -def test_generate_sell_reason_stats(default_conf): +def test_generate_sell_reason_stats(): results = pd.DataFrame( { @@ -280,9 +289,10 @@ def test_generate_sell_reason_stats(default_conf): assert stop_result['profit_mean_pct'] == round(stop_result['profit_mean'] * 100, 2) -def test_text_table_strategy(default_conf, mocker): +def test_text_table_strategy(default_conf): + default_conf['max_open_trades'] = 2 results = {} - results['TestStrategy1'] = pd.DataFrame( + results['TestStrategy1'] = {'results': pd.DataFrame( { 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], 'profit_percent': [0.1, 0.2, 0.3], @@ -293,8 +303,8 @@ def test_text_table_strategy(default_conf, mocker): 'losses': [0, 0, 1], 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] } - ) - results['TestStrategy2'] = pd.DataFrame( + ), 'config': default_conf} + results['TestStrategy2'] = {'results': pd.DataFrame( { 'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'], 'profit_percent': [0.4, 0.2, 0.3], @@ -305,7 +315,7 @@ def test_text_table_strategy(default_conf, mocker): 'losses': [0, 0, 1], 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] } - ) + ), 'config': default_conf} result_str = ( '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' @@ -318,14 +328,12 @@ def test_text_table_strategy(default_conf, mocker): ' 45.00 | 0:20:00 | 3 | 0 | 0 |' ) - strategy_results = generate_strategy_metrics(stake_currency='BTC', - max_open_trades=2, - all_results=results) + strategy_results = generate_strategy_metrics(all_results=results) assert text_table_strategy(strategy_results, 'BTC') == result_str -def test_generate_edge_table(edge_conf, mocker): +def test_generate_edge_table(): results = {} results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 9217abc46..1f05bef1e 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -231,9 +231,6 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # VolumePairList only ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), - # Different sorting depending on quote or bid volume - ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']), # No pair for ETH, VolumePairList @@ -263,10 +260,6 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), - # Precisionfilter bid - ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, - {"method": "PrecisionFilter"}], - "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], @@ -293,9 +286,6 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "StaticPairList"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), # Static Pairlist before VolumePairList - sorting changes - ([{"method": "StaticPairList"}, - {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['HOT/BTC', 'TKN/BTC', 'ETH/BTC']), # SpreadFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}], @@ -344,9 +334,9 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "SpreadFilter", "max_spread_ratio": 0.005}], "BTC", 'filter_at_the_beginning'), # OperationalException expected # Static Pairlist after VolumePairList, on a non-first position - ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "StaticPairList"}], - "BTC", 'static_in_the_middle'), + "BTC", 'static_in_the_middle'), ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.02}], "USDT", ['ETH/USDT', 'NANO/USDT']), diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index c370dce8f..c2dee6439 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -313,7 +313,6 @@ def test_rpc_delete_trade(mocker, default_conf, fee, markets, caplog): with pytest.raises(RPCException, match='invalid argument'): rpc._rpc_delete('200') - create_mock_trades(fee) trades = Trade.query.all() trades[1].stoploss_order_id = '1234' trades[2].stoploss_order_id = '1234' @@ -669,7 +668,8 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: return_value={ 'status': 'closed', 'type': 'limit', - 'side': 'buy' + 'side': 'buy', + 'filled': 0.0, } ), get_fee=fee, @@ -695,6 +695,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: msg = rpc._rpc_forcesell('all') assert msg == {'result': 'Created sell orders for all open trades.'} + freqtradebot.enter_positions() msg = rpc._rpc_forcesell('1') assert msg == {'result': 'Created sell order for trade 1.'} @@ -707,17 +708,26 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 + freqtradebot.enter_positions() # make an limit-buy open trade trade = Trade.query.filter(Trade.id == '1').first() filled_amount = trade.amount / 2 + # Fetch order - it's open first, and closed after cancel_order is called. mocker.patch( 'freqtrade.exchange.Exchange.fetch_order', - return_value={ + side_effect=[{ + 'id': '1234', 'status': 'open', 'type': 'limit', 'side': 'buy', 'filled': filled_amount - } + }, { + 'id': '1234', + 'status': 'closed', + 'type': 'limit', + 'side': 'buy', + 'filled': filled_amount + }] ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called # and trade amount is updated @@ -725,6 +735,16 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: assert cancel_order_mock.call_count == 1 assert trade.amount == filled_amount + mocker.patch( + 'freqtrade.exchange.Exchange.fetch_order', + return_value={ + 'status': 'open', + 'type': 'limit', + 'side': 'buy', + 'filled': filled_amount + }) + + freqtradebot.config['max_open_trades'] = 3 freqtradebot.enter_positions() trade = Trade.query.filter(Trade.id == '2').first() amount = trade.amount @@ -744,20 +764,22 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: assert cancel_order_mock.call_count == 2 assert trade.amount == amount - freqtradebot.enter_positions() # make an limit-sell open trade mocker.patch( 'freqtrade.exchange.Exchange.fetch_order', return_value={ 'status': 'open', 'type': 'limit', - 'side': 'sell' + 'side': 'sell', + 'amount': amount, + 'remaining': amount, + 'filled': 0.0 } ) msg = rpc._rpc_forcesell('3') assert msg == {'result': 'Created sell order for trade 3.'} # status quo, no exchange calls - assert cancel_order_mock.call_count == 2 + assert cancel_order_mock.call_count == 3 def test_performance_handle(default_conf, ticker, limit_buy_order, fee, @@ -816,10 +838,10 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: assert counts["current"] == 1 -def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None: +def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order_open) -> None: default_conf['forcebuy_enable'] = True mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) + buy_mm = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index d2b69ee4f..626586a4a 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -266,7 +266,7 @@ def test_api_reloadconf(botclient): rc = client_post(client, f"{BASE_URI}/reload_config") assert_response(rc) - assert rc.json == {'status': 'reloading config ...'} + assert rc.json == {'status': 'Reloading config ...'} assert ftbot.state == State.RELOAD_CONFIG @@ -435,7 +435,7 @@ def test_api_logs(botclient): assert len(rc.json) == 2 assert 'logs' in rc.json # Using a fixed comparison here would make this test fail! - assert rc.json['log_count'] > 10 + assert rc.json['log_count'] > 1 assert len(rc.json['logs']) == rc.json['log_count'] assert isinstance(rc.json['logs'][0], list) @@ -471,6 +471,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): assert rc.json == {"error": "Error querying _edge: Edge is not enabled."} +@pytest.mark.usefixtures("init_persistence") def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, limit_sell_order): ftbot, client = botclient patch_get_signal(ftbot, (True, False)) @@ -498,6 +499,7 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li assert rc.json['best_pair'] == '' assert rc.json['best_rate'] == 0 + trade = Trade.query.first() trade.update(limit_sell_order) trade.close_date = datetime.utcnow() diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index edf6bae4d..e8d0f648e 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -124,10 +124,10 @@ def test_send_msg_webhook_CustomMessagetype(mocker, default_conf, caplog) -> Non rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] - rpc_manager.send_msg({'type': RPCMessageType.CUSTOM_NOTIFICATION, + rpc_manager.send_msg({'type': RPCMessageType.STARTUP_NOTIFICATION, 'status': 'TestMessage'}) assert log_has( - "Message type RPCMessageType.CUSTOM_NOTIFICATION not implemented by handler webhook.", + "Message type 'startup' not implemented by handler webhook.", caplog) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index c962f68db..3958a825a 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -14,6 +14,7 @@ from telegram import Chat, Message, Update from telegram.error import NetworkError from freqtrade import __version__ +from freqtrade.constants import CANCEL_REASON from freqtrade.edge import PairInfo from freqtrade.freqtradebot import FreqtradeBot from freqtrade.loggers import setup_logging @@ -252,7 +253,6 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, ) msg_mock = MagicMock() @@ -691,7 +691,7 @@ def test_reload_config_handle(default_conf, update, mocker) -> None: telegram._reload_config(update=update, context=MagicMock()) assert freqtradebot.state == State.RELOAD_CONFIG assert msg_mock.call_count == 1 - assert 'reloading config' in msg_mock.call_args_list[0][0][0] + assert 'Reloading config' in msg_mock.call_args_list[0][0][0] def test_telegram_forcesell_handle(default_conf, update, ticker, fee, @@ -725,7 +725,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, context.args = ["1"] telegram._forcesell(update=update, context=context) - assert rpc_mock.call_count == 2 + assert rpc_mock.call_count == 3 last_msg = rpc_mock.call_args_list[-1][0][0] assert { 'type': RPCMessageType.SELL_NOTIFICATION, @@ -784,7 +784,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, context.args = ["1"] telegram._forcesell(update=update, context=context) - assert rpc_mock.call_count == 2 + assert rpc_mock.call_count == 3 last_msg = rpc_mock.call_args_list[-1][0][0] assert { @@ -834,8 +834,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None context.args = ["all"] telegram._forcesell(update=update, context=context) - assert rpc_mock.call_count == 4 - msg = rpc_mock.call_args_list[0][0][0] + # Called for each trade 3 times + assert rpc_mock.call_count == 8 + msg = rpc_mock.call_args_list[1][0][0] assert { 'type': RPCMessageType.SELL_NOTIFICATION, 'trade_id': 1, @@ -1005,7 +1006,6 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -1299,16 +1299,14 @@ def test_show_config_handle(default_conf, update, mocker) -> None: assert '*Initial Stoploss:* `-0.1`' in msg_mock.call_args_list[0][0][0] -def test_send_msg_buy_notification(default_conf, mocker) -> None: +def test_send_msg_buy_notification(default_conf, mocker, caplog) -> None: msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _send_msg=msg_mock ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) - telegram.send_msg({ + msg = { 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': 'Bittrex', 'pair': 'ETH/BTC', @@ -1321,7 +1319,10 @@ def test_send_msg_buy_notification(default_conf, mocker) -> None: 'current_rate': 1.099e-05, 'amount': 1333.3333333333335, 'open_date': arrow.utcnow().shift(hours=-1) - }) + } + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg(msg) assert msg_mock.call_args[0][0] \ == '\N{LARGE BLUE CIRCLE} *Bittrex:* Buying ETH/BTC\n' \ '*Amount:* `1333.33333333`\n' \ @@ -1329,6 +1330,21 @@ def test_send_msg_buy_notification(default_conf, mocker) -> None: '*Current Rate:* `0.00001099`\n' \ '*Total:* `(0.001000 BTC, 12.345 USD)`' + freqtradebot.config['telegram']['notification_settings'] = {'buy': 'off'} + caplog.clear() + msg_mock.reset_mock() + telegram.send_msg(msg) + msg_mock.call_count == 0 + log_has("Notification 'buy' not sent.", caplog) + + freqtradebot.config['telegram']['notification_settings'] = {'buy': 'silent'} + caplog.clear() + msg_mock.reset_mock() + + telegram.send_msg(msg) + msg_mock.call_count == 1 + msg_mock.call_args_list[0][1]['disable_notification'] is True + def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: msg_mock = MagicMock() @@ -1343,9 +1359,10 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, 'exchange': 'Bittrex', 'pair': 'ETH/BTC', + 'reason': CANCEL_REASON['TIMEOUT'] }) - assert msg_mock.call_args[0][0] \ - == ('\N{WARNING SIGN} *Bittrex:* Cancelling Open Buy Order for ETH/BTC') + assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Bittrex:* ' + 'Cancelling open buy Order for ETH/BTC. Reason: cancelled due to timeout.') def test_send_msg_sell_notification(default_conf, mocker) -> None: @@ -1484,7 +1501,7 @@ def test_warning_notification(default_conf, mocker) -> None: assert msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Warning:* `message`' -def test_custom_notification(default_conf, mocker) -> None: +def test_startup_notification(default_conf, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -1494,7 +1511,7 @@ def test_custom_notification(default_conf, mocker) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) telegram.send_msg({ - 'type': RPCMessageType.CUSTOM_NOTIFICATION, + 'type': RPCMessageType.STARTUP_NOTIFICATION, 'status': '*Custom:* `Hello World`' }) assert msg_mock.call_args[0][0] == '*Custom:* `Hello World`' diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 1ced62746..9256a5316 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -150,7 +150,7 @@ def test_send_msg(default_conf, mocker): default_conf["webhook"]["webhooksellcancel"]["value3"].format(**msg)) for msgtype in [RPCMessageType.STATUS_NOTIFICATION, RPCMessageType.WARNING_NOTIFICATION, - RPCMessageType.CUSTOM_NOTIFICATION]: + RPCMessageType.STARTUP_NOTIFICATION]: # Test notification msg = { 'type': msgtype, @@ -174,7 +174,7 @@ def test_exception_send_msg(default_conf, mocker, caplog): webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) webhook.send_msg({'type': RPCMessageType.BUY_NOTIFICATION}) - assert log_has(f"Message type {RPCMessageType.BUY_NOTIFICATION} not configured for webhooks", + assert log_has(f"Message type '{RPCMessageType.BUY_NOTIFICATION}' not configured for webhooks", caplog) default_conf["webhook"] = get_webhook_dict() diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py new file mode 100644 index 000000000..4b29bf304 --- /dev/null +++ b/tests/strategy/test_strategy_helpers.py @@ -0,0 +1,88 @@ +import pandas as pd +import numpy as np + +from freqtrade.strategy import merge_informative_pair, timeframe_to_minutes + + +def generate_test_data(timeframe: str, size: int): + np.random.seed(42) + tf_mins = timeframe_to_minutes(timeframe) + + base = np.random.normal(20, 2, size=size) + + date = pd.period_range('2020-07-05', periods=size, freq=f'{tf_mins}min').to_timestamp() + df = pd.DataFrame({ + 'date': date, + 'open': base, + 'high': base + np.random.normal(2, 1, size=size), + 'low': base - np.random.normal(2, 1, size=size), + 'close': base + np.random.normal(0, 1, size=size), + 'volume': np.random.normal(200, size=size) + } + ) + df = df.dropna() + return df + + +def test_merge_informative_pair(): + data = generate_test_data('15m', 40) + informative = generate_test_data('1h', 40) + + result = merge_informative_pair(data, informative, '15m', '1h', ffill=True) + assert isinstance(result, pd.DataFrame) + assert len(result) == len(data) + assert 'date' in result.columns + assert result['date'].equals(data['date']) + assert 'date_1h' in result.columns + + assert 'open' in result.columns + assert 'open_1h' in result.columns + assert result['open'].equals(data['open']) + + assert 'close' in result.columns + assert 'close_1h' in result.columns + assert result['close'].equals(data['close']) + + assert 'volume' in result.columns + assert 'volume_1h' in result.columns + assert result['volume'].equals(data['volume']) + + # First 4 rows are empty + assert result.iloc[0]['date_1h'] is pd.NaT + assert result.iloc[1]['date_1h'] is pd.NaT + assert result.iloc[2]['date_1h'] is pd.NaT + assert result.iloc[3]['date_1h'] is pd.NaT + # Next 4 rows contain the starting date (0:00) + assert result.iloc[4]['date_1h'] == result.iloc[0]['date'] + assert result.iloc[5]['date_1h'] == result.iloc[0]['date'] + assert result.iloc[6]['date_1h'] == result.iloc[0]['date'] + assert result.iloc[7]['date_1h'] == result.iloc[0]['date'] + # Next 4 rows contain the next Hourly date original date row 4 + assert result.iloc[8]['date_1h'] == result.iloc[4]['date'] + + +def test_merge_informative_pair_same(): + data = generate_test_data('15m', 40) + informative = generate_test_data('15m', 40) + + result = merge_informative_pair(data, informative, '15m', '15m', ffill=True) + assert isinstance(result, pd.DataFrame) + assert len(result) == len(data) + assert 'date' in result.columns + assert result['date'].equals(data['date']) + assert 'date_15m' in result.columns + + assert 'open' in result.columns + assert 'open_15m' in result.columns + assert result['open'].equals(data['open']) + + assert 'close' in result.columns + assert 'close_15m' in result.columns + assert result['close'].equals(data['close']) + + assert 'volume' in result.columns + assert 'volume_15m' in result.columns + assert result['volume'].equals(data['volume']) + + # Dates match 1:1 + assert result['date_15m'].equals(result['date']) diff --git a/tests/test_docs.sh b/tests/test_docs.sh index 8a354daad..09e142b99 100755 --- a/tests/test_docs.sh +++ b/tests/test_docs.sh @@ -2,8 +2,7 @@ # Test Documentation boxes - # !!! : is not allowed! # !!! "title" - Title needs to be quoted! -# !!! Spaces at the beginning are not allowed -grep -Er '^!{3}\s\S+:|^!{3}\s\S+\s[^"]|^\s+!{3}\s\S+' docs/* +grep -Er '^!{3}\s\S+:|^!{3}\s\S+\s[^"]' docs/* if [ $? -ne 0 ]; then echo "Docs test success." diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 0d7968e26..0c12c05bb 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -13,10 +13,12 @@ import pytest from freqtrade.constants import (CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT) from freqtrade.exceptions import (DependencyException, ExchangeError, + InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, TemporaryError) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade +from freqtrade.persistence.models import Order from freqtrade.rpc import RPCMessageType from freqtrade.state import RunMode, State from freqtrade.strategy.interface import SellCheckTuple, SellType @@ -25,6 +27,11 @@ from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal, patch_wallet, patch_whitelist) +from tests.conftest_trades import (MOCK_TRADE_COUNT, mock_order_1, + mock_order_2, mock_order_2_sell, + mock_order_3, mock_order_3_sell, + mock_order_4, mock_order_5_stoploss, + mock_order_6_sell) def patch_RPCManager(mocker) -> MagicMock: @@ -170,7 +177,7 @@ def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: (True, 0.0027, 3, 0.5, [0.001, 0.001, 0.000673]), (True, 0.0022, 3, 1, [0.001, 0.001, 0.0]), ]) -def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order, +def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order_open, amend_last, wallet, max_open, lsamr, expected) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -178,7 +185,7 @@ def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_b 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee ) default_conf['dry_run_wallet'] = wallet @@ -191,6 +198,7 @@ def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_b for i in range(0, max_open): if expected[i] is not None: + limit_buy_order_open['id'] = str(i) result = freqtrade.get_trade_stake_amount('ETH/BTC') assert pytest.approx(result) == expected[i] freqtrade.execute_buy('ETH/BTC', result) @@ -216,13 +224,13 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None: (0.50, 0.0025), ]) def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, balance_ratio, result1, - limit_buy_order, fee, mocker) -> None: + limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee ) @@ -303,7 +311,6 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf 'ask': buy_price * 0.79, 'last': buy_price * 0.79 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) ############################################# @@ -343,7 +350,6 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, 'ask': buy_price * 0.85, 'last': buy_price * 0.85 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) ############################################# @@ -362,8 +368,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, assert freqtrade.handle_trade(trade) is False -def test_total_open_trades_stakes(mocker, default_conf, ticker, - limit_buy_order, fee) -> None: +def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf['stake_amount'] = 0.00098751 @@ -371,7 +376,6 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -534,7 +538,6 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -568,7 +571,6 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -578,11 +580,11 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, freqtrade.create_trade('ETH/BTC') -def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, +def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) + buy_mock = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -598,11 +600,11 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, assert rate * amount <= default_conf['stake_amount'] -def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order, +def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) + buy_mock = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -618,14 +620,14 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord assert not freqtrade.create_trade('ETH/BTC') -def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: +def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order_open, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_balance=MagicMock(return_value=default_conf['stake_amount']), get_fee=fee, ) @@ -639,14 +641,14 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, assert freqtrade.get_trade_stake_amount('ETH/BTC') == 0 -def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order, fee, +def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -702,7 +704,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: @pytest.mark.parametrize("max_open", range(0, 5)) @pytest.mark.parametrize("tradable_balance_ratio,modifier", [(1.0, 1), (0.99, 0.8), (0.5, 0.5)]) -def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, +def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, limit_buy_order_open, max_open, tradable_balance_ratio, modifier) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -713,7 +715,7 @@ def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': "12355555"}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -727,14 +729,14 @@ def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, assert len(trades) == max(int(max_open * modifier), 0) -def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: +def test_create_trades_preopen(default_conf, ticker, fee, mocker, limit_buy_order_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf['max_open_trades'] = 4 mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': "12355555"}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -745,6 +747,8 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: freqtrade.execute_buy('NEO/BTC', default_conf['stake_amount']) assert len(Trade.get_open_trades()) == 2 + # Change order_id for new orders + limit_buy_order_open['id'] = '123444' # Create 2 new trades using create_trades assert freqtrade.create_trade('ETH/BTC') @@ -754,14 +758,14 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: assert len(trades) == 4 -def test_process_trade_creation(default_conf, ticker, limit_buy_order, +def test_process_trade_creation(default_conf, ticker, limit_buy_order, limit_buy_order_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), fetch_order=MagicMock(return_value=limit_buy_order), get_fee=fee, ) @@ -824,14 +828,14 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None: assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] -def test_process_trade_handling(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_process_trade_handling(default_conf, ticker, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - fetch_order=MagicMock(return_value=limit_buy_order), + buy=MagicMock(return_value=limit_buy_order_open), + fetch_order=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -971,7 +975,7 @@ def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, assert not log_has("Using cached buy rate for ETH/BTC.", caplog) -def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: +def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) freqtrade = FreqtradeBot(default_conf) @@ -984,7 +988,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: get_buy_rate=buy_rate_mock, _get_min_pair_stake_amount=MagicMock(return_value=1) ) - buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) + buy_mm = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ @@ -1003,6 +1007,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: assert freqtrade.strategy.confirm_trade_entry.call_count == 1 buy_rate_mock.reset_mock() + limit_buy_order_open['id'] = '22' freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True) assert freqtrade.execute_buy(pair, stake_amount) assert buy_rate_mock.call_count == 1 @@ -1018,9 +1023,10 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: trade = Trade.query.first() assert trade assert trade.is_open is True - assert trade.open_order_id == limit_buy_order['id'] + assert trade.open_order_id == '22' # Test calling with price + limit_buy_order_open['id'] = '33' fix_price = 0.06 assert freqtrade.execute_buy(pair, stake_amount, fix_price) # Make sure get_buy_rate wasn't called again @@ -1036,6 +1042,8 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: limit_buy_order['status'] = 'closed' limit_buy_order['price'] = 10 limit_buy_order['cost'] = 100 + limit_buy_order['id'] = '444' + mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order)) assert freqtrade.execute_buy(pair, stake_amount) trade = Trade.query.all()[2] @@ -1051,11 +1059,12 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: limit_buy_order['remaining'] = 10.00 limit_buy_order['price'] = 0.5 limit_buy_order['cost'] = 40.495905365 + limit_buy_order['id'] = '555' mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order)) assert freqtrade.execute_buy(pair, stake_amount) trade = Trade.query.all()[3] assert trade - assert trade.open_order_id is None + assert trade.open_order_id == '555' assert trade.open_rate == 0.5 assert trade.stake_amount == 40.495905365 @@ -1066,6 +1075,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: limit_buy_order['remaining'] = 90.99181073 limit_buy_order['price'] = 0.5 limit_buy_order['cost'] = 0.0 + limit_buy_order['id'] = '66' mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order)) assert not freqtrade.execute_buy(pair, stake_amount) @@ -1093,9 +1103,11 @@ def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) - freqtrade.strategy.confirm_trade_entry = MagicMock(side_effect=ValueError) assert freqtrade.execute_buy(pair, stake_amount) + limit_buy_order['id'] = '222' freqtrade.strategy.confirm_trade_entry = MagicMock(side_effect=Exception) assert freqtrade.execute_buy(pair, stake_amount) + limit_buy_order['id'] = '2223' freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True) assert freqtrade.execute_buy(pair, stake_amount) @@ -1201,6 +1213,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, assert trade stoploss_order_hit = MagicMock(return_value={ + 'id': 100, 'status': 'closed', 'type': 'stop_loss_limit', 'price': 3, @@ -1209,7 +1222,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, }) mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hit) assert freqtrade.handle_stoploss_on_exchange(trade) is True - assert log_has('STOP_LOSS_LIMIT is hit for {}.'.format(trade), caplog) + assert log_has_re(r'STOP_LOSS_LIMIT is hit for Trade\(id=1, .*\)\.', caplog) assert trade.stoploss_order_id is None assert trade.is_open is False @@ -1258,7 +1271,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - fetch_stoploss_order=MagicMock(return_value={'status': 'canceled'}), + fetch_stoploss_order=MagicMock(return_value={'status': 'canceled', 'id': 100}), stoploss=MagicMock(side_effect=ExchangeError()), ) freqtrade = FreqtradeBot(default_conf) @@ -1278,7 +1291,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, - limit_buy_order, limit_sell_order): + limit_buy_order_open, limit_sell_order): rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) sell_mock = MagicMock(return_value={'id': limit_sell_order['id']}) @@ -1289,7 +1302,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, 'ask': 0.00001173, 'last': 0.00001172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), sell=sell_mock, get_fee=fee, fetch_order=MagicMock(return_value={'status': 'canceled'}), @@ -1320,7 +1333,46 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market' -def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, +def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, fee, + limit_buy_order_open, limit_sell_order): + sell_mock = MagicMock(return_value={'id': limit_sell_order['id']}) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + mock_insuf = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_insufficient_funds') + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=MagicMock(return_value=limit_buy_order_open), + sell=sell_mock, + get_fee=fee, + fetch_order=MagicMock(return_value={'status': 'canceled'}), + stoploss=MagicMock(side_effect=InsufficientFundsError()), + ) + patch_get_signal(freqtrade) + freqtrade.strategy.order_types['stoploss_on_exchange'] = True + + freqtrade.enter_positions() + trade = Trade.query.first() + caplog.clear() + freqtrade.create_stoploss_order(trade, 200) + # stoploss_orderid was empty before + assert trade.stoploss_order_id is None + assert mock_insuf.call_count == 1 + mock_insuf.reset_mock() + + trade.stoploss_order_id = 'stoploss_orderid' + freqtrade.create_stoploss_order(trade, 200) + # No change to stoploss-orderid + assert trade.stoploss_order_id == 'stoploss_orderid' + assert mock_insuf.call_count == 1 + + +@pytest.mark.usefixtures("init_persistence") +def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set stoploss = MagicMock(return_value={'id': 13434334}) @@ -1389,7 +1441,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, })) cancel_order_mock = MagicMock() - stoploss_order_mock = MagicMock() + stoploss_order_mock = MagicMock(return_value={'id': 13434334}) mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', cancel_order_mock) mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss_order_mock) @@ -1691,8 +1743,10 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No open_date=arrow.utcnow().datetime, amount=11, ) + assert not freqtrade.update_trade_state(trade, None) + assert log_has_re(r'Orderid for trade .* is empty.', caplog) # Add datetime explicitly since sqlalchemy defaults apply only once written to database - freqtrade.update_trade_state(trade) + freqtrade.update_trade_state(trade, '123') # Test amount not modified by fee-logic assert not log_has_re(r'Applying fee to .*', caplog) assert trade.open_order_id is None @@ -1702,14 +1756,14 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81) assert trade.amount != 90.81 # test amount modified by fee-logic - freqtrade.update_trade_state(trade) + freqtrade.update_trade_state(trade, '123') assert trade.amount == 90.81 assert trade.open_order_id is None trade.is_open = True trade.open_order_id = None # Assert we call handle_trade() if trade is feasible for execution - freqtrade.update_trade_state(trade) + freqtrade.update_trade_state(trade, '123') assert log_has_re('Found open order for.*', caplog) @@ -1734,7 +1788,7 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ open_order_id="123456", is_open=True, ) - freqtrade.update_trade_state(trade, limit_buy_order) + freqtrade.update_trade_state(trade, '123456', limit_buy_order) assert trade.amount != amount assert trade.amount == limit_buy_order['amount'] @@ -1756,11 +1810,11 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_ open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456", + open_order_id='123456', is_open=True, open_date=arrow.utcnow().datetime, ) - freqtrade.update_trade_state(trade, limit_buy_order) + freqtrade.update_trade_state(trade, '123456', limit_buy_order) assert trade.amount != amount assert trade.amount == limit_buy_order['amount'] assert log_has_re(r'Applying fee on amount for .*', caplog) @@ -1780,7 +1834,7 @@ def test_update_trade_state_exception(mocker, default_conf, 'freqtrade.freqtradebot.FreqtradeBot.get_real_amount', side_effect=DependencyException() ) - freqtrade.update_trade_state(trade) + freqtrade.update_trade_state(trade, trade.open_order_id) assert log_has('Could not update trade amount: ', caplog) @@ -1795,12 +1849,13 @@ def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None # Test raise of OperationalException exception grm_mock = mocker.patch("freqtrade.freqtradebot.FreqtradeBot.get_real_amount", MagicMock()) - freqtrade.update_trade_state(trade) + freqtrade.update_trade_state(trade, trade.open_order_id) assert grm_mock.call_count == 0 assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog) -def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order, mocker): +def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order_open, + limit_sell_order, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) # fetch_order should not be called!! mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError)) @@ -1823,14 +1878,20 @@ def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_orde open_order_id="123456", is_open=True, ) - freqtrade.update_trade_state(trade, limit_sell_order) + order = Order.parse_from_ccxt_object(limit_sell_order_open, 'LTC/ETH', 'sell') + trade.orders.append(order) + assert order.status == 'open' + freqtrade.update_trade_state(trade, trade.open_order_id, limit_sell_order) assert trade.amount == limit_sell_order['amount'] # Wallet needs to be updated after closing a limit-sell order to reenable buying assert wallet_mock.call_count == 1 assert not trade.is_open + # Order is updated by update_trade_state + assert order.status == 'closed' -def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mocker) -> None: +def test_handle_trade(default_conf, limit_buy_order, limit_sell_order_open, limit_sell_order, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -1840,8 +1901,8 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock 'ask': 0.00001173, 'last': 0.00001172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - sell=MagicMock(return_value={'id': limit_sell_order['id']}), + buy=MagicMock(return_value=limit_buy_order), + sell=MagicMock(return_value=limit_sell_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -1870,13 +1931,14 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock assert trade.close_date is not None -def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order_open, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -1921,7 +1983,7 @@ def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order, fee, assert freqtrade.handle_trade(trades[0]) is True -def test_handle_trade_roi(default_conf, ticker, limit_buy_order, +def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open, fee, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) @@ -1929,7 +1991,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -1954,14 +2016,14 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, def test_handle_trade_use_sell_signal( - default_conf, ticker, limit_buy_order, fee, mocker, caplog) -> None: + default_conf, ticker, limit_buy_order_open, fee, mocker, caplog) -> None: # use_sell_signal is True buy default caplog.set_level(logging.DEBUG) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -1982,14 +2044,14 @@ def test_handle_trade_use_sell_signal( caplog) -def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, +def test_close_trade(default_conf, ticker, limit_buy_order, limit_buy_order_open, limit_sell_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -2289,7 +2351,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old # note this is for a partially-complete buy order freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 2 + assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() assert len(trades) == 1 assert trades[0].amount == 23.0 @@ -2324,7 +2386,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap assert log_has_re(r"Applying fee on amount for Trade.*", caplog) assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 2 + assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() assert len(trades) == 1 # Verify that trade has been updated @@ -2364,7 +2426,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, assert log_has_re(r"Could not update trade amount: .*", caplog) assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 2 + assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() assert len(trades) == 1 # Verify that trade has been updated @@ -2527,13 +2589,15 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: send_msg_mock.reset_mock() order['amount'] = 2 - assert freqtrade.handle_cancel_sell(trade, order, reason) == CANCEL_REASON['PARTIALLY_FILLED'] + assert freqtrade.handle_cancel_sell(trade, order, reason + ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] # Assert cancel_order was not called (callcount remains unchanged) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 - assert freqtrade.handle_cancel_sell(trade, order, reason) == CANCEL_REASON['PARTIALLY_FILLED'] + assert freqtrade.handle_cancel_sell(trade, order, reason + ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] # Message should not be iterated again - assert trade.sell_order_status == CANCEL_REASON['PARTIALLY_FILLED'] + assert trade.sell_order_status == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] assert send_msg_mock.call_count == 1 @@ -2805,7 +2869,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, fee, - limit_buy_order, mocker) -> None: + mocker) -> None: default_conf['exchange']['name'] = 'binance' rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) @@ -2927,7 +2991,36 @@ def test_execute_sell_market_order(default_conf, ticker, fee, } == last_msg -def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, +def test_execute_sell_insufficient_funds_error(default_conf, ticker, fee, + ticker_sell_up, mocker) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mock_insuf = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_insufficient_funds') + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + get_fee=fee, + sell=MagicMock(side_effect=InsufficientFundsError()) + ) + patch_get_signal(freqtrade) + + # Create some test data + freqtrade.enter_positions() + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker_sell_up + ) + + assert not freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], + sell_reason=SellType.ROI) + assert mock_insuf.call_count == 1 + + +def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2938,7 +3031,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, 'ask': 0.00002173, 'last': 0.00002172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -2959,7 +3052,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, +def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2970,7 +3063,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, 'ask': 0.00002173, 'last': 0.00002172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -2990,7 +3083,8 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker) -> None: +def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, limit_buy_order_open, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3000,7 +3094,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker 'ask': 0.00000173, 'last': 0.00000172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -3019,7 +3113,8 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker assert freqtrade.handle_trade(trade) is False -def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocker) -> None: +def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, limit_buy_order_open, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3029,7 +3124,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke 'ask': 0.0000173, 'last': 0.0000172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -3051,7 +3146,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_sell_not_enough_balance(default_conf, limit_buy_order, +def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3062,7 +3157,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, 'ask': 0.00002173, 'last': 0.00002172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -3170,7 +3265,8 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo assert log_has(f"Pair {trade.pair} is currently locked.", caplog) -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3180,7 +3276,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> 'ask': 0.0000173, 'last': 0.0000172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -3204,7 +3300,8 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> assert trade.sell_reason == SellType.ROI.value -def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: +def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order, + fee, caplog, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3214,7 +3311,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) 'ask': 0.00001099, 'last': 0.00001099 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['trailing_stop'] = True @@ -3254,7 +3351,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, +def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_order_open, fee, caplog, mocker) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) @@ -3266,7 +3363,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, 'ask': buy_price - 0.000001, 'last': buy_price - 0.000001 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['trailing_stop'] = True @@ -3311,7 +3408,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) -def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, +def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_order_open, fee, caplog, mocker) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) @@ -3323,7 +3420,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, 'ask': buy_price - 0.000001, 'last': buy_price - 0.000001 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -3369,7 +3466,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, +def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_open, fee, caplog, mocker) -> None: buy_price = limit_buy_order['price'] # buy_price: 0.00001099 @@ -3383,7 +3480,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, 'ask': buy_price, 'last': buy_price }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -3432,7 +3529,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, assert trade.stop_loss == 0.0000117705 -def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, +def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3443,7 +3540,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, 'ask': 0.00000173, 'last': 0.00000172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -3785,8 +3882,8 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker, assert walletmock.call_count == 1 -def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker, - order_book_l2): +def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, limit_buy_order, + fee, mocker, order_book_l2): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 patch_RPCManager(mocker) @@ -3795,7 +3892,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -3910,8 +4007,8 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: assert freqtrade._check_depth_of_market_buy('ETH/BTC', conf) is False -def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order, - fee, mocker, order_book_l2, caplog) -> None: +def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_order, fee, + limit_sell_order_open, mocker, order_book_l2, caplog) -> None: """ test order book ask strategy """ @@ -3930,8 +4027,8 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order 'ask': 0.00001173, 'last': 0.00001172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - sell=MagicMock(return_value={'id': limit_sell_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), + sell=MagicMock(return_value=limit_sell_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -4076,7 +4173,7 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): @pytest.mark.usefixtures("init_persistence") -def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, caplog): +def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_open, caplog): default_conf['dry_run'] = True # Initialize to 2 times stake amount default_conf['dry_run_wallet'] = 0.002 @@ -4086,7 +4183,7 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -4111,21 +4208,22 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order, limit_sell_order): default_conf['cancel_open_orders_on_exit'] = True mocker.patch('freqtrade.exchange.Exchange.fetch_order', - side_effect=[ExchangeError(), limit_sell_order, limit_buy_order]) + side_effect=[ + ExchangeError(), limit_sell_order, limit_buy_order, limit_sell_order]) buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_buy') sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_sell') freqtrade = get_patched_freqtradebot(mocker, default_conf) create_mock_trades(fee) trades = Trade.query.all() - assert len(trades) == 4 + assert len(trades) == MOCK_TRADE_COUNT freqtrade.cancel_all_open_orders() assert buy_mock.call_count == 1 - assert sell_mock.call_count == 1 + assert sell_mock.call_count == 2 @pytest.mark.usefixtures("init_persistence") -def test_check_for_open_trades(mocker, default_conf, fee, limit_buy_order, limit_sell_order): +def test_check_for_open_trades(mocker, default_conf, fee): freqtrade = get_patched_freqtradebot(mocker, default_conf) freqtrade.check_for_open_trades() @@ -4138,3 +4236,246 @@ def test_check_for_open_trades(mocker, default_conf, fee, limit_buy_order, limit freqtrade.check_for_open_trades() assert freqtrade.rpc.send_msg.call_count == 1 assert 'Handle these trades manually' in freqtrade.rpc.send_msg.call_args[0][0]['status'] + + +@pytest.mark.usefixtures("init_persistence") +def test_update_open_orders(mocker, default_conf, fee, caplog): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + create_mock_trades(fee) + + freqtrade.update_open_orders() + assert log_has_re(r"Error updating Order .*", caplog) + caplog.clear() + + assert len(Order.get_open_orders()) == 3 + matching_buy_order = mock_order_4() + matching_buy_order.update({ + 'status': 'closed', + }) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=matching_buy_order) + freqtrade.update_open_orders() + # Only stoploss and sell orders are kept open + assert len(Order.get_open_orders()) == 2 + + +@pytest.mark.usefixtures("init_persistence") +def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + def patch_with_fee(order): + order.update({'fee': {'cost': 0.1, 'rate': 0.2, + 'currency': order['symbol'].split('/')[0]}}) + return order + + mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + side_effect=[ + patch_with_fee(mock_order_2_sell()), + patch_with_fee(mock_order_3_sell()), + patch_with_fee(mock_order_1()), + patch_with_fee(mock_order_2()), + patch_with_fee(mock_order_3()), + patch_with_fee(mock_order_4()), + ] + ) + + create_mock_trades(fee) + trades = Trade.get_trades().all() + assert len(trades) == MOCK_TRADE_COUNT + for trade in trades: + assert trade.fee_open_cost is None + assert trade.fee_open_currency is None + assert trade.fee_close_cost is None + assert trade.fee_close_currency is None + + freqtrade.update_closed_trades_without_assigned_fees() + + trades = Trade.get_trades().all() + assert len(trades) == MOCK_TRADE_COUNT + + for trade in trades: + if trade.is_open: + # Exclude Trade 4 - as the order is still open. + if trade.select_order('buy', False): + assert trade.fee_open_cost is not None + assert trade.fee_open_currency is not None + else: + assert trade.fee_open_cost is None + assert trade.fee_open_currency is None + + else: + assert trade.fee_close_cost is not None + assert trade.fee_close_currency is not None + + +@pytest.mark.usefixtures("init_persistence") +def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state') + + create_mock_trades(fee) + trades = Trade.get_trades().all() + + freqtrade.reupdate_buy_order_fees(trades[0]) + assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) + assert mock_uts.call_count == 1 + assert mock_uts.call_args_list[0][0][0] == trades[0] + assert mock_uts.call_args_list[0][0][1] == mock_order_1()['id'] + assert log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) + mock_uts.reset_mock() + caplog.clear() + + # Test with trade without orders + trade = Trade( + pair='XRP/ETH', + stake_amount=0.001, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_date=arrow.utcnow().datetime, + is_open=True, + amount=20, + open_rate=0.01, + exchange='bittrex', + ) + Trade.session.add(trade) + + freqtrade.reupdate_buy_order_fees(trade) + assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) + assert mock_uts.call_count == 0 + assert not log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) + + +@pytest.mark.usefixtures("init_persistence") +def test_handle_insufficient_funds(mocker, default_conf, fee): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mock_rlo = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.refind_lost_order') + mock_bof = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.reupdate_buy_order_fees') + create_mock_trades(fee) + trades = Trade.get_trades().all() + + # Trade 0 has only a open buy order, no closed order + freqtrade.handle_insufficient_funds(trades[0]) + assert mock_rlo.call_count == 0 + assert mock_bof.call_count == 1 + + mock_rlo.reset_mock() + mock_bof.reset_mock() + + # Trade 1 has closed buy and sell orders + freqtrade.handle_insufficient_funds(trades[1]) + assert mock_rlo.call_count == 1 + assert mock_bof.call_count == 0 + + mock_rlo.reset_mock() + mock_bof.reset_mock() + + # Trade 2 has closed buy and sell orders + freqtrade.handle_insufficient_funds(trades[2]) + assert mock_rlo.call_count == 1 + assert mock_bof.call_count == 0 + + mock_rlo.reset_mock() + mock_bof.reset_mock() + + # Trade 3 has an opne buy order + freqtrade.handle_insufficient_funds(trades[3]) + assert mock_rlo.call_count == 0 + assert mock_bof.call_count == 1 + + +@pytest.mark.usefixtures("init_persistence") +def test_refind_lost_order(mocker, default_conf, fee, caplog): + caplog.set_level(logging.DEBUG) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state') + + mock_fo = mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + return_value={'status': 'open'}) + + def reset_open_orders(trade): + trade.open_order_id = None + trade.stoploss_order_id = None + + create_mock_trades(fee) + trades = Trade.get_trades().all() + + caplog.clear() + + # No open order + trade = trades[0] + reset_open_orders(trade) + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + freqtrade.refind_lost_order(trade) + order = mock_order_1() + assert log_has_re(r"Order Order(.*order_id=" + order['id'] + ".*) is no longer open.", caplog) + assert mock_fo.call_count == 0 + assert mock_uts.call_count == 0 + # No change to orderid - as update_trade_state is mocked + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + caplog.clear() + mock_fo.reset_mock() + + # Open buy order + trade = trades[3] + reset_open_orders(trade) + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + freqtrade.refind_lost_order(trade) + order = mock_order_4() + assert log_has_re(r"Trying to refind Order\(.*", caplog) + assert mock_fo.call_count == 0 + assert mock_uts.call_count == 0 + # No change to orderid - as update_trade_state is mocked + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + caplog.clear() + mock_fo.reset_mock() + + # Open stoploss order + trade = trades[4] + reset_open_orders(trade) + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + freqtrade.refind_lost_order(trade) + order = mock_order_5_stoploss() + assert log_has_re(r"Trying to refind Order\(.*", caplog) + assert mock_fo.call_count == 1 + assert mock_uts.call_count == 1 + # stoploss_order_id is "refound" and added to the trade + assert trade.open_order_id is None + assert trade.stoploss_order_id is not None + + caplog.clear() + mock_fo.reset_mock() + mock_uts.reset_mock() + + # Open sell order + trade = trades[5] + reset_open_orders(trade) + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + freqtrade.refind_lost_order(trade) + order = mock_order_6_sell() + assert log_has_re(r"Trying to refind Order\(.*", caplog) + assert mock_fo.call_count == 1 + assert mock_uts.call_count == 1 + # sell-orderid is "refound" and added to the trade + assert trade.open_order_id == order['id'] + assert trade.stoploss_order_id is None + + caplog.clear() + + # Test error case + mock_fo = mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + side_effect=ExchangeError()) + order = mock_order_5_stoploss() + + freqtrade.refind_lost_order(trades[4]) + assert log_has(f"Error updating {order['id']}.", caplog) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 65c83e05b..adfa18876 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -7,9 +7,9 @@ import pytest from sqlalchemy import create_engine from freqtrade import constants -from freqtrade.exceptions import OperationalException -from freqtrade.persistence import Trade, clean_dry_run_db, init -from tests.conftest import log_has, create_mock_trades +from freqtrade.exceptions import DependencyException, OperationalException +from freqtrade.persistence import Order, Trade, clean_dry_run_db, init +from tests.conftest import create_mock_trades, log_has, log_has_re def test_init_create_session(default_conf): @@ -22,7 +22,7 @@ def test_init_create_session(default_conf): def test_init_custom_db_url(default_conf, mocker): # Update path to a value other than default, but still in-memory default_conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'}) - create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock()) init(default_conf['db_url'], default_conf['dry_run']) assert create_engine_mock.call_count == 1 @@ -40,7 +40,7 @@ def test_init_prod_db(default_conf, mocker): default_conf.update({'dry_run': False}) default_conf.update({'db_url': constants.DEFAULT_DB_PROD_URL}) - create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock()) init(default_conf['db_url'], default_conf['dry_run']) assert create_engine_mock.call_count == 1 @@ -51,7 +51,7 @@ def test_init_dryrun_db(default_conf, mocker): default_conf.update({'dry_run': True}) default_conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL}) - create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock()) init(default_conf['db_url'], default_conf['dry_run']) assert create_engine_mock.call_count == 1 @@ -93,6 +93,8 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog): stake_amount=0.001, open_rate=0.01, amount=5, + is_open=True, + open_date=arrow.utcnow().datetime, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', @@ -107,9 +109,9 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog): assert trade.open_rate == 0.00001099 assert trade.close_profit is None assert trade.close_date is None - assert log_has("LIMIT_BUY has been fulfilled for Trade(id=2, " - "pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=closed).", - caplog) + assert log_has_re(r"LIMIT_BUY has been fulfilled for Trade\(id=2, " + r"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=.*\).", + caplog) caplog.clear() trade.open_order_id = 'something' @@ -118,9 +120,9 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog): assert trade.close_rate == 0.00001173 assert trade.close_profit == 0.06201058 assert trade.close_date is not None - assert log_has("LIMIT_SELL has been fulfilled for Trade(id=2, " - "pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=closed).", - caplog) + assert log_has_re(r"LIMIT_SELL has been fulfilled for Trade\(id=2, " + r"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=.*\).", + caplog) @pytest.mark.usefixtures("init_persistence") @@ -131,8 +133,10 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog): stake_amount=0.001, amount=5, open_rate=0.01, + is_open=True, fee_open=fee.return_value, fee_close=fee.return_value, + open_date=arrow.utcnow().datetime, exchange='bittrex', ) @@ -142,20 +146,21 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog): assert trade.open_rate == 0.00004099 assert trade.close_profit is None assert trade.close_date is None - assert log_has("MARKET_BUY has been fulfilled for Trade(id=1, " - "pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=closed).", - caplog) + assert log_has_re(r"MARKET_BUY has been fulfilled for Trade\(id=1, " + r"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=.*\).", + caplog) caplog.clear() + trade.is_open = True trade.open_order_id = 'something' trade.update(market_sell_order) assert trade.open_order_id is None assert trade.close_rate == 0.00004173 assert trade.close_profit == 0.01297561 assert trade.close_date is not None - assert log_has("MARKET_SELL has been fulfilled for Trade(id=1, " - "pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=closed).", - caplog) + assert log_has_re(r"MARKET_SELL has been fulfilled for Trade\(id=1, " + r"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=.*\).", + caplog) @pytest.mark.usefixtures("init_persistence") @@ -184,6 +189,36 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee): assert trade.calc_profit_ratio() == 0.06201058 +@pytest.mark.usefixtures("init_persistence") +def test_trade_close(limit_buy_order, limit_sell_order, fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + open_rate=0.01, + amount=5, + is_open=True, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_date=arrow.Arrow(2020, 2, 1, 15, 5, 1).datetime, + exchange='bittrex', + ) + assert trade.close_profit is None + assert trade.close_date is None + assert trade.is_open is True + trade.close(0.02) + assert trade.is_open is False + assert trade.close_profit == 0.99002494 + assert trade.close_date is not None + + new_date = arrow.Arrow(2020, 2, 2, 15, 6, 1).datetime, + assert trade.close_date != new_date + # Close should NOT update close_date if the trade has been closed already + assert trade.is_open is False + trade.close_date = new_date + trade.close(0.02) + assert trade.close_date == new_date + + @pytest.mark.usefixtures("init_persistence") def test_calc_close_trade_price_exception(limit_buy_order, fee): trade = Trade( @@ -421,9 +456,9 @@ def test_migrate_old(mocker, default_conf, fee): PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" - insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, + insert_table_old = """INSERT INTO trades (exchange, pair, is_open, open_order_id, fee, open_rate, stake_amount, amount, open_date) - VALUES ('BITTREX', 'BTC_ETC', 1, {fee}, + VALUES ('BITTREX', 'BTC_ETC', 1, '123123', {fee}, 0.00258580, {stake}, {amount}, '2017-11-28 12:44:24.000000') """.format(fee=fee.return_value, @@ -440,7 +475,7 @@ def test_migrate_old(mocker, default_conf, fee): amount=amount ) engine = create_engine('sqlite://') - mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format engine.execute(create_table_old) @@ -481,6 +516,12 @@ def test_migrate_old(mocker, default_conf, fee): assert pytest.approx(trade.close_profit_abs) == trade.calc_profit() assert trade.sell_order_status is None + # Should've created one order + assert len(Order.query.all()) == 1 + order = Order.query.first() + assert order.order_id == '123123' + assert order.ft_order_side == 'buy' + def test_migrate_new(mocker, default_conf, fee, caplog): """ @@ -509,22 +550,25 @@ def test_migrate_new(mocker, default_conf, fee, caplog): sell_reason VARCHAR, strategy VARCHAR, ticker_interval INTEGER, + stoploss_order_id VARCHAR, PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date, - stop_loss, initial_stop_loss, max_rate, ticker_interval) + stop_loss, initial_stop_loss, max_rate, ticker_interval, + open_order_id, stoploss_order_id) VALUES ('binance', 'ETC/BTC', 1, {fee}, 0.00258580, {stake}, {amount}, '2019-11-28 12:44:24.000000', - 0.0, 0.0, 0.0, '5m') + 0.0, 0.0, 0.0, '5m', + 'buy_order', 'stop_order_id222') """.format(fee=fee.return_value, stake=default_conf.get("stake_amount"), amount=amount ) engine = create_engine('sqlite://') - mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format engine.execute(create_table_old) @@ -558,14 +602,23 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.sell_reason is None assert trade.strategy is None assert trade.timeframe == '5m' - assert trade.stoploss_order_id is None + assert trade.stoploss_order_id == 'stop_order_id222' assert trade.stoploss_last_update is None assert log_has("trying trades_bak1", caplog) assert log_has("trying trades_bak2", caplog) - assert log_has("Running database migration - backup available as trades_bak2", caplog) + assert log_has("Running database migration for trades - backup: trades_bak2", caplog) assert trade.open_trade_price == trade._calc_open_trade_price() assert trade.close_profit_abs is None + assert log_has("Moving open orders to Orders table.", caplog) + orders = Order.query.all() + assert len(orders) == 2 + assert orders[0].order_id == 'buy_order' + assert orders[0].ft_order_side == 'buy' + + assert orders[1].order_id == 'stop_order_id222' + assert orders[1].ft_order_side == 'stoploss' + def test_migrate_mid_state(mocker, default_conf, fee, caplog): """ @@ -601,7 +654,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): amount=amount ) engine = create_engine('sqlite://') - mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format engine.execute(create_table_old) @@ -626,7 +679,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): assert trade.initial_stop_loss == 0.0 assert trade.open_trade_price == trade._calc_open_trade_price() assert log_has("trying trades_bak0", caplog) - assert log_has("Running database migration - backup available as trades_bak0", caplog) + assert log_has("Running database migration for trades - backup: trades_bak0", caplog) def test_adjust_stop_loss(fee): @@ -713,10 +766,10 @@ def test_adjust_min_max_rates(fee): @pytest.mark.usefixtures("init_persistence") -def test_get_open(default_conf, fee): +def test_get_open(fee): create_mock_trades(fee) - assert len(Trade.get_open_trades()) == 2 + assert len(Trade.get_open_trades()) == 4 @pytest.mark.usefixtures("init_persistence") @@ -986,7 +1039,7 @@ def test_total_open_trades_stakes(fee): assert res == 0 create_mock_trades(fee) res = Trade.total_open_trades_stakes() - assert res == 0.002 + assert res == 0.004 @pytest.mark.usefixtures("init_persistence") @@ -1012,3 +1065,96 @@ def test_get_best_pair(fee): assert len(res) == 2 assert res[0] == 'XRP/BTC' assert res[1] == 0.01 + + +@pytest.mark.usefixtures("init_persistence") +def test_update_order_from_ccxt(): + # Most basic order return (only has orderid) + o = Order.parse_from_ccxt_object({'id': '1234'}, 'ETH/BTC', 'buy') + assert isinstance(o, Order) + assert o.ft_pair == 'ETH/BTC' + assert o.ft_order_side == 'buy' + assert o.order_id == '1234' + assert o.ft_is_open + ccxt_order = { + 'id': '1234', + 'side': 'buy', + 'symbol': 'ETH/BTC', + 'type': 'limit', + 'price': 1234.5, + 'amount': 20.0, + 'filled': 9, + 'remaining': 11, + 'status': 'open', + 'timestamp': 1599394315123 + } + o = Order.parse_from_ccxt_object(ccxt_order, 'ETH/BTC', 'buy') + assert isinstance(o, Order) + assert o.ft_pair == 'ETH/BTC' + assert o.ft_order_side == 'buy' + assert o.order_id == '1234' + assert o.order_type == 'limit' + assert o.price == 1234.5 + assert o.filled == 9 + assert o.remaining == 11 + assert o.order_date is not None + assert o.ft_is_open + assert o.order_filled_date is None + + # Order has been closed + ccxt_order.update({'filled': 20.0, 'remaining': 0.0, 'status': 'closed'}) + o.update_from_ccxt_object(ccxt_order) + + assert o.filled == 20.0 + assert o.remaining == 0.0 + assert not o.ft_is_open + assert o.order_filled_date is not None + + ccxt_order.update({'id': 'somethingelse'}) + with pytest.raises(DependencyException, match=r"Order-id's don't match"): + o.update_from_ccxt_object(ccxt_order) + + +@pytest.mark.usefixtures("init_persistence") +def test_select_order(fee): + create_mock_trades(fee) + + trades = Trade.get_trades().all() + + # Open buy order, no sell order + order = trades[0].select_order('buy', True) + assert order is None + order = trades[0].select_order('buy', False) + assert order is not None + order = trades[0].select_order('sell', None) + assert order is None + + # closed buy order, and open sell order + order = trades[1].select_order('buy', True) + assert order is None + order = trades[1].select_order('buy', False) + assert order is not None + order = trades[1].select_order('buy', None) + assert order is not None + order = trades[1].select_order('sell', True) + assert order is None + order = trades[1].select_order('sell', False) + assert order is not None + + # Has open buy order + order = trades[3].select_order('buy', True) + assert order is not None + order = trades[3].select_order('buy', False) + assert order is None + + # Open sell order + order = trades[4].select_order('buy', True) + assert order is None + order = trades[4].select_order('buy', False) + assert order is not None + + order = trades[4].select_order('sell', True) + assert order is not None + assert order.ft_order_side == 'stoploss' + order = trades[4].select_order('sell', False) + assert order is None diff --git a/tests/test_wallets.py b/tests/test_wallets.py index 884470014..450dabc4d 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -19,12 +19,17 @@ def test_sync_wallet_at_boot(mocker, default_conf): "used": 0.0, "total": 0.260739 }, + "USDT": { + "free": 20, + "used": 20, + "total": 40 + }, }) ) freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert len(freqtrade.wallets._wallets) == 2 + assert len(freqtrade.wallets._wallets) == 3 assert freqtrade.wallets._wallets['BNT'].free == 1.0 assert freqtrade.wallets._wallets['BNT'].used == 2.0 assert freqtrade.wallets._wallets['BNT'].total == 3.0 @@ -32,6 +37,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets._wallets['GAS'].used == 0.0 assert freqtrade.wallets._wallets['GAS'].total == 0.260739 assert freqtrade.wallets.get_free('BNT') == 1.0 + assert 'USDT' in freqtrade.wallets._wallets assert freqtrade.wallets._last_wallet_refresh > 0 mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -51,6 +57,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): freqtrade.wallets.update() + # USDT is missing from the 2nd result - so should not be in this either. assert len(freqtrade.wallets._wallets) == 2 assert freqtrade.wallets._wallets['BNT'].free == 1.2 assert freqtrade.wallets._wallets['BNT'].used == 1.9 diff --git a/tests/testdata/UNITTEST_BTC-5m.h5 b/tests/testdata/UNITTEST_BTC-5m.h5 new file mode 100644 index 000000000..52232af9e Binary files /dev/null and b/tests/testdata/UNITTEST_BTC-5m.h5 differ diff --git a/tests/testdata/XRP_ETH-trades.h5 b/tests/testdata/XRP_ETH-trades.h5 new file mode 100644 index 000000000..c13789e2a Binary files /dev/null and b/tests/testdata/XRP_ETH-trades.h5 differ