diff --git a/.coveragerc b/.coveragerc index 4bd5b63fa..96ad6b09b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,6 @@ [run] omit = scripts/* - freqtrade/tests/* freqtrade/vendor/* freqtrade/__main__.py + tests/* diff --git a/.gitignore b/.gitignore index 1664ad7eb..9ac2c9d5d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,9 @@ # Freqtrade rules -freqtrade/tests/testdata/*.json -hyperopt_conf.py config*.json *.sqlite -.hyperopt logfile.txt -hyperopt_trials.pickle user_data/* +!user_data/strategy/sample_strategy.py !user_data/notebooks user_data/notebooks/* !user_data/notebooks/*example.ipynb diff --git a/.travis.yml b/.travis.yml index c75049276..405228ab8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,19 +22,19 @@ jobs: include: - stage: tests script: - - pytest --random-order --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ + - pytest --random-order --cov=freqtrade --cov-config=.coveragerc # Allow failure for coveralls - coveralls || true name: pytest - script: - cp config.json.example config.json - - freqtrade --datadir freqtrade/tests/testdata backtesting + - freqtrade --datadir tests/testdata backtesting name: backtest - script: - cp config.json.example config.json - - freqtrade --datadir freqtrade/tests/testdata hyperopt -e 5 + - freqtrade --datadir tests/testdata hyperopt -e 5 name: hyperopt - - script: flake8 freqtrade scripts + - script: flake8 name: flake8 - script: # Test Documentation boxes - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e15059f56..72c04e151 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ Few pointers for contributions: - Create your PR against the `develop` branch, not `master`. - New features need to contain unit tests and must be PEP8 conformant (max-line-length = 100). -If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg) +If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. ## Getting started @@ -28,19 +28,19 @@ make it pass. It means you have introduced a regression. #### Test the whole project ```bash -pytest freqtrade +pytest ``` #### Test only one file ```bash -pytest freqtrade/tests/test_.py +pytest tests/test_.py ``` #### Test only one method from one file ```bash -pytest freqtrade/tests/test_.py::test_ +pytest tests/test_.py::test_ ``` ### 2. Test if your code is PEP8 compliant diff --git a/Dockerfile.pi b/Dockerfile.pi index 1b9c4c579..85ba5892f 100644 --- a/Dockerfile.pi +++ b/Dockerfile.pi @@ -22,13 +22,13 @@ RUN tar -xzf /freqtrade/ta-lib-0.4.0-src.tar.gz \ ENV LD_LIBRARY_PATH /usr/local/lib # Install berryconda -RUN wget https://github.com/jjhelmus/berryconda/releases/download/v2.0.0/Berryconda3-2.0.0-Linux-armv7l.sh \ +RUN wget -q https://github.com/jjhelmus/berryconda/releases/download/v2.0.0/Berryconda3-2.0.0-Linux-armv7l.sh \ && bash ./Berryconda3-2.0.0-Linux-armv7l.sh -b \ && rm Berryconda3-2.0.0-Linux-armv7l.sh # Install dependencies COPY requirements-common.txt /freqtrade/ -RUN ~/berryconda3/bin/conda install -y numpy pandas scipy \ +RUN ~/berryconda3/bin/conda install -y numpy pandas \ && ~/berryconda3/bin/pip install -r requirements-common.txt --no-cache-dir # Install and execute diff --git a/MANIFEST.in b/MANIFEST.in index 63508c05d..7529152a0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,4 +2,3 @@ include LICENSE include README.md include config.json.example recursive-include freqtrade *.py -include freqtrade/tests/testdata/*.json diff --git a/README.md b/README.md index 240b4f917..6d57dcd89 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Accounts having BNB accounts use this to pay for fees - if your first trade happ For any questions not covered by the documentation or for further information about the bot, we encourage you to join our slack channel. -- [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg). +- [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE). ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) @@ -172,7 +172,7 @@ to understand the requirements before sending your pull-requests. Coding is not a neccessity to contribute - maybe start with improving our documentation? Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. -**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. +**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. **Important:** Always create your PR against the `develop` branch, not `master`. diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 7a8127c44..839ca0876 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -23,7 +23,7 @@ if [ $? -ne 0 ]; then fi # Run backtest -docker run --rm -it -v $(pwd)/config.json.example:/freqtrade/config.json:ro freqtrade:${TAG} --datadir freqtrade/tests/testdata backtesting +docker run --rm -it -v $(pwd)/config.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} --datadir /tests/testdata backtesting if [ $? -ne 0 ]; then echo "failed running backtest" diff --git a/config_full.json.example b/config_full.json.example index b6451859c..957967042 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -38,6 +38,7 @@ "order_types": { "buy": "limit", "sell": "limit", + "emergencysell": "market", "stoploss": "market", "stoploss_on_exchange": false, "stoploss_on_exchange_interval": 60 diff --git a/docs/assets/plot-dataframe.png b/docs/assets/plot-dataframe.png new file mode 100644 index 000000000..eb90a1734 Binary files /dev/null and b/docs/assets/plot-dataframe.png differ diff --git a/docs/assets/plot-profit.png b/docs/assets/plot-profit.png new file mode 100644 index 000000000..88d69a2d4 Binary files /dev/null and b/docs/assets/plot-profit.png differ diff --git a/docs/backtesting.md b/docs/backtesting.md index 13d19f0ca..75aba6c73 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -1,41 +1,9 @@ # Backtesting -This page explains how to validate your strategy performance by using -Backtesting. +This page explains how to validate your strategy performance by using Backtesting. -## Getting data for backtesting and hyperopt - -To download data (candles / OHLCV) needed for backtesting and hyperoptimization use the `freqtrade download-data` command. - -If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes. -Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. - -Alternatively, a `pairs.json` file can be used. - -If you are using Binance for example: - -- create a directory `user_data/data/binance` and copy `pairs.json` in that directory. -- update the `pairs.json` to contain the currency pairs you are interested in. - -```bash -mkdir -p user_data/data/binance -cp freqtrade/tests/testdata/pairs.json user_data/data/binance -``` - -Then run: - -```bash -freqtrade download-data --exchange binance -``` - -This will download ticker data for all the currency pairs you defined in `pairs.json`. - -- To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. -- To change the exchange used to download the tickers, please use a different configuration file (you'll probably need to adjust ratelimits etc.) -- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. -- To download ticker data for only 10 days, use `--days 10` (defaults to 30 days). -- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. -- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. +Backtesting requires historic data to be available. +To learn how to get data for the pairs and exchange you're interested in, head over to the [Data Downloading](data-download.md) section of the documentation. ## Test your strategy with Backtesting @@ -43,18 +11,16 @@ Now you have good Buy and Sell strategies and some historic data, you want to te real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). -Backtesting will use the crypto-currencies (pair) from your config file -and load static tickers located in -[/freqtrade/tests/testdata](https://github.com/freqtrade/freqtrade/tree/develop/freqtrade/tests/testdata). -If the 5 min and 1 min ticker for the crypto-currencies to test is not -already in the `testdata` directory, backtesting will download them -automatically. Testdata files will not be updated until you specify it. +Backtesting will use the crypto-currencies (pairs) from your config file +and load ticker data from `user_data/data/` by default. +If no data is available for the exchange / pair / ticker interval combination, backtesting will +ask you to download them first using `freqtrade download-data`. +For details on downloading, please refer to the [Data Downloading](data-download.md) section in the documentation. -The result of backtesting will confirm you if your bot has better odds of making a profit than a loss. - -The backtesting is very easy with freqtrade. +The result of backtesting will confirm if your bot has better odds of making a profit than a loss. ### Run a backtesting against the currencies listed in your config file + #### With 5 min tickers (Per default) ```bash @@ -79,18 +45,18 @@ freqtrade backtesting --datadir user_data/data/bittrex-20180101 #### With a (custom) strategy file ```bash -freqtrade -s TestStrategy backtesting +freqtrade -s SampleStrategy backtesting ``` -Where `-s TestStrategy` refers to the class name within the strategy file `test_strategy.py` found in the `freqtrade/user_data/strategies` directory. +Where `-s SampleStrategy` refers to the class name within the strategy file `sample_strategy.py` found in the `freqtrade/user_data/strategies` directory. #### Comparing multiple Strategies ```bash -freqtrade backtesting --strategy-list TestStrategy1 AwesomeStrategy --ticker-interval 5m +freqtrade backtesting --strategy-list SampleStrategy1 AwesomeStrategy --ticker-interval 5m ``` -Where `TestStrategy1` and `AwesomeStrategy` refer to class names of strategies. +Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies. #### Exporting trades to file @@ -103,36 +69,35 @@ The exported trades can be used for [further analysis](#further-backtest-result- #### Exporting trades to file specifying a custom filename ```bash -freqtrade backtesting --export trades --export-filename=backtest_teststrategy.json +freqtrade backtesting --export trades --export-filename=backtest_samplestrategy.json ``` -#### Running backtest with smaller testset +#### Running backtest with smaller testset by using timerange -Use the `--timerange` argument to change how much of the testset -you want to use. The last N ticks/timeframes will be used. +Use the `--timerange` argument to change how much of the testset you want to use. -Example: + +For example, running backtesting with the `--timerange=20190501-` option will use all available data starting with May 1st, 2019 from your inputdata. ```bash -freqtrade backtesting --timerange=-200 +freqtrade backtesting --timerange=20190501- ``` -#### Advanced use of timerange - -Doing `--timerange=-200` will get the last 200 timeframes -from your inputdata. You can also specify specific dates, -or a range span indexed by start and stop. +You can also specify particular dates or a range span indexed by start and stop. The full timerange specification: -- Use last 123 tickframes of data: `--timerange=-123` -- Use first 123 tickframes of data: `--timerange=123-` -- Use tickframes from line 123 through 456: `--timerange=123-456` - Use tickframes till 2018/01/31: `--timerange=-20180131` - Use tickframes since 2018/01/31: `--timerange=20180131-` - Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` - Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` +- Use last 123 tickframes of data: `--timerange=-123` +- Use first 123 tickframes of data: `--timerange=123-` +- Use tickframes from line 123 through 456: `--timerange=123-456` + +!!! warning + Be carefull when using non-date functions - these do not allow you to specify precise dates, so if you updated the test-data it will probably use a different dataset. ## Understand the backtesting result @@ -178,11 +143,12 @@ A backtesting result will look like that: | TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 | 0 | ``` -The 1st table will contain all trades the bot made. +The 1st table contains all trades the bot made, including "left open trades". -The 2nd table will contain a recap of sell reasons. +The 2nd table contains a recap of sell reasons. -The 3rd table will contain all trades the bot had to `forcesell` at the end of the backtest period to present a full picture. +The 3rd table contains all trades the bot had to `forcesell` at the end of the backtest period to present a full picture. +This is necessary to simulate realistic behaviour, since the backtest period has to end at some point, while realistically, you could leave the bot running forever. These trades are also included in the first table, but are extracted separately for clarity. The last line will give you the overall performance of your strategy, @@ -192,22 +158,16 @@ here: | TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 243 | ``` -We understand the bot has made `429` trades for an average duration of -`4:12:00`, with a performance of `76.20%` (profit), that means it has +The bot has made `429` trades for an average duration of `4:12:00`, with a performance of `76.20%` (profit), that means it has earned a total of `0.00762792 BTC` starting with a capital of 0.01 BTC. -The column `avg profit %` shows the average profit for all trades made while the column `cum profit %` sums all the profits/losses. -The column `tot profit %` shows instead the total profit % in relation to allocated capital -(`max_open_trades * stake_amount`). In the above results we have `max_open_trades=2 stake_amount=0.005` in config -so `(76.20/100) * (0.005 * 2) =~ 0.00762792 BTC`. +The column `avg profit %` shows the average profit for all trades made while the column `cum profit %` sums up all the profits/losses. +The column `tot profit %` shows instead the total profit % in relation to allocated capital (`max_open_trades * stake_amount`). +In the above results we have `max_open_trades=2` and `stake_amount=0.005` in config so `tot_profit %` will be `(76.20/100) * (0.005 * 2) =~ 0.00762792 BTC`. -As you will see your strategy performance will be influenced by your buy -strategy, your sell strategy, and also by the `minimal_roi` and -`stop_loss` you have set. +Your strategy performance is influenced by your buy strategy, your sell strategy, and also by the `minimal_roi` and `stop_loss` you have set. -As for an example if your minimal_roi is only `"0": 0.01`. You cannot -expect the bot to make more profit than 1% (because it will sell every -time a trade will reach 1%). +For example, if your `minimal_roi` is only `"0": 0.01` you cannot expect the bot to make more profit than 1% (because it will sell every time a trade reaches 1%). ```json "minimal_roi": { @@ -216,22 +176,33 @@ time a trade will reach 1%). ``` On the other hand, if you set a too high `minimal_roi` like `"0": 0.55` -(55%), there is a lot of chance that the bot will never reach this -profit. Hence, keep in mind that your performance is a mix of your -strategies, your configuration, and the crypto-currency you have set up. +(55%), there is almost no chance that the bot will ever reach this profit. +Hence, keep in mind that your performance is an integral mix of all different elements of the strategy, your configuration, and the crypto-currency pairs you have set up. + +### Assumptions made by backtesting + +Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions: + +- Buys happen at open-price +- Low happens before high for stoploss, protecting capital first. +- ROI sells are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the sell will be at 2%) +- Stoploss sells happen exactly at stoploss price, even if low was lower +- Trailing stoploss + - High happens first - adjusting stoploss + - Low uses the adjusted stoploss (so sells with large high-low difference are backtested correctly) +- Sell-reason does not explain if a trade was positive or negative, just what triggered the sell (this can look odd if negative ROI values are used) ### Further backtest-result analysis To further analyze your backtest results, you can [export the trades](#exporting-trades-to-file). You can then load the trades to perform further analysis as shown in our [data analysis](data-analysis.md#backtesting) backtesting section. - ## Backtesting multiple strategies -To backtest multiple strategies, a list of Strategies can be provided. +To compare multiple strategies, a list of Strategies can be provided to backtesting. This is limited to 1 ticker-interval per run, however, data is only loaded once from disk so if you have multiple -strategies you'd like to compare, this should give a nice runtime boost. +strategies you'd like to compare, this will give a nice runtime boost. All listed Strategies need to be in the same directory. @@ -241,7 +212,7 @@ freqtrade backtesting --timerange 20180401-20180410 --ticker-interval 5m --strat This will save the results to `user_data/backtest_results/backtest-result-.json`, injecting the strategy-name into the target filename. There will be an additional table comparing win/losses of the different strategies (identical to the "Total" row in the first table). -Detailed output for all strategies one after the other will be available, so make sure to scroll up. +Detailed output for all strategies one after the other will be available, so make sure to scroll up to see the details per strategy. ``` =========================================================== Strategy Summary =========================================================== diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 0b0561d3d..f44400e32 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -184,10 +184,6 @@ optional arguments: Specify max_open_trades to use. --stake_amount STAKE_AMOUNT Specify stake_amount. - -r, --refresh-pairs-cached - Refresh the pairs files in tests/testdata with the - latest data from the exchange. Use it if you want to - run your optimization commands with up-to-date data. --eps, --enable-position-stacking Allow buying the same pair multiple times (position stacking). @@ -245,10 +241,6 @@ optional arguments: Specify max_open_trades to use. --stake_amount STAKE_AMOUNT Specify stake_amount. - -r, --refresh-pairs-cached - Refresh the pairs files in tests/testdata with the - latest data from the exchange. Use it if you want to - run your optimization commands with up-to-date data. --customhyperopt NAME Specify hyperopt class name (default: `DefaultHyperOpts`). @@ -310,10 +302,6 @@ optional arguments: Specify max_open_trades to use. --stake_amount STAKE_AMOUNT Specify stake_amount. - -r, --refresh-pairs-cached - Refresh the pairs files in tests/testdata with the - latest data from the exchange. Use it if you want to - run your optimization commands with up-to-date data. --stoplosses STOPLOSS_RANGE Defines a range of stoploss against which edge will assess the strategy the format is "min,max,step" diff --git a/docs/configuration.md b/docs/configuration.md index 2d21de942..0d902766a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,14 +1,15 @@ # Configure the bot -This page explains how to configure the bot. +Freqtrade has many configurable features and possibilities. +By default, these settings are configured via the configuration file (see below). ## The Freqtrade configuration file The bot uses a set of configuration parameters during its operation that all together conform the bot configuration. It normally reads its configuration from a file (Freqtrade configuration file). -Per default, the bot loads configuration from the `config.json` file located in the current working directory. +Per default, the bot loads the configuration from the `config.json` file, located in the current working directory. -You can change the name of the configuration file used by the bot with the `-c/--config` command line option. +You can specify a different configuration file used by the bot with the `-c/--config` command line option. In some advanced use cases, multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream. @@ -22,19 +23,26 @@ The Freqtrade configuration file is to be written in the JSON format. Additionally to the standard JSON syntax, you may use one-line `// ...` and multi-line `/* ... */` comments in your configuration files and trailing commas in the lists of parameters. -Do not worry if you are not familiar with JSON format -- simply open the configuration file with an editor of your choice, make some changes to the parameters you need, save your changes and, finally, restart the bot or, if it was previously stopped, run it again with the changes you made to the configuration. The bot validates syntax of the configuration file at startup and will warn you if you made any errors editing it. +Do not worry if you are not familiar with JSON format -- simply open the configuration file with an editor of your choice, make some changes to the parameters you need, save your changes and, finally, restart the bot or, if it was previously stopped, run it again with the changes you made to the configuration. The bot validates syntax of the configuration file at startup and will warn you if you made any errors editing it, pointing out problematic lines. ## Configuration parameters The table below will list all configuration parameters available. -Mandatory parameters are marked as **Required**. +Freqtrade can also load many options via command line (CLI) arguments (check out the commands `--help` output for details). +The prevelance for all Options is as follows: + +- CLI arguments override any other option +- Configuration files are used in sequence (last file wins), and override Strategy configurations. +- Strategy configurations are only used if they are not set via configuration or via command line arguments. These options are market with [Strategy Override](#parameters-in-the-strategy) in the below table. + +Mandatory parameters are marked as **Required**, which means that they are required to be set in one of the possible ways. | Command | Default | Description | |----------|---------|-------------| | `max_open_trades` | 3 | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades) -| `stake_currency` | BTC | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy). -| `stake_amount` | 0.05 | **Required.** Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to `"unlimited"` to allow the bot to use all available balance. [Strategy Override](#parameters-in-the-strategy). +| `stake_currency` | BTC | **Required.** Crypto-currency used for trading. +| `stake_amount` | 0.05 | **Required.** Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to `"unlimited"` to allow the bot to use all available balance. | `amount_reserve_percent` | 0.05 | Reserve some amount in min pair stake amount. Default is 5%. The bot will reserve `amount_reserve_percent` + stop-loss value when calculating min pair stake amount in order to avoid possible trade refusals. | `ticker_interval` | [1m, 5m, 15m, 30m, 1h, 1d, ...] | The ticker interval to use (1min, 5 min, 15 min, 30 min, 1 hour or 1 day). Default is 5 minutes. [Strategy Override](#parameters-in-the-strategy). | `fiat_display_currency` | USD | **Required.** Fiat currency used to show your profits. More information below. @@ -61,8 +69,9 @@ Mandatory parameters are marked as **Required**. | `order_time_in_force` | None | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy). | `exchange.name` | | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.sandbox` | false | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details. -| `exchange.key` | '' | API key to use for the exchange. Only required when you are in production mode. -| `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. +| `exchange.key` | '' | API key to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** +| `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** +| `exchange.password` | '' | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. ***Keep it in secrete, do not disclose publicly.*** | `exchange.pair_whitelist` | [] | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Can be overriden by dynamic pairlists (see [below](#dynamic-pairlists)). | `exchange.pair_blacklist` | [] | List of pairs the bot must absolutely avoid for trading and backtesting. Can be overriden by dynamic pairlists (see [below](#dynamic-pairlists)). | `exchange.ccxt_config` | None | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) @@ -76,8 +85,8 @@ Mandatory parameters are marked as **Required**. | `pairlist.method` | StaticPairList | Use static or dynamic volume-based pairlist. [More information below](#dynamic-pairlists). | `pairlist.config` | None | Additional configuration for dynamic pairlists. [More information below](#dynamic-pairlists). | `telegram.enabled` | true | **Required.** Enable or not the usage of Telegram. -| `telegram.token` | token | Your Telegram bot token. Only required if `telegram.enabled` is `true`. -| `telegram.chat_id` | chat_id | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. +| `telegram.token` | token | Your Telegram bot token. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.*** +| `telegram.chat_id` | chat_id | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.*** | `webhook.enabled` | false | Enable usage of Webhook notifications | `webhook.url` | false | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. | `webhook.webhookbuy` | false | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. @@ -98,8 +107,6 @@ Mandatory parameters are marked as **Required**. The following parameters can be set in either configuration file or strategy. Values set in the configuration file always overwrite values set in the strategy. -* `stake_currency` -* `stake_amount` * `ticker_interval` * `minimal_roi` * `stoploss` @@ -191,19 +198,20 @@ end up paying more then would probably have been necessary. ### Understand order_types -The `order_types` configuration parameter contains a dict mapping order-types to -market-types as well as stoploss on or off exchange type and stoploss on exchange -update interval in seconds. This allows to buy using limit orders, sell using -limit-orders, and create stoploss orders using market. It also allows to set the -stoploss "on exchange" which means stoploss order would be placed immediately once -the buy order is fulfilled. In case stoploss on exchange and `trailing_stop` are -both set, then the bot will use `stoploss_on_exchange_interval` to check it periodically -and update it if necessary (e.x. in case of trailing stoploss). -This can be set in the configuration file or in the strategy. -Values set in the configuration file overwrites values set in the strategy. +The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds. -If this is configured, all 4 values (`buy`, `sell`, `stoploss` and -`stoploss_on_exchange`) need to be present, otherwise the bot will warn about it and fail to start. +This allows to buy using limit orders, sell using +limit-orders, and create stoplosses using using market orders. It also allows to set the +stoploss "on exchange" which means stoploss order would be placed immediately once +the buy order is fulfilled. +If `stoploss_on_exchange` and `trailing_stop` are both set, then the bot will use `stoploss_on_exchange_interval` to check and update the stoploss on exchange periodically. +`order_types` can be set in the configuration file or in the strategy. +`order_types` set in the configuration file overwrites values set in the strategy as a whole, so you need to configure the whole `order_types` dictionary in one place. + +If this is configured, the following 4 values (`buy`, `sell`, `stoploss` and +`stoploss_on_exchange`) need to be present, otherwise the bot will fail to start. + +`emergencysell` is an optional value, which defaults to `market` and is used when creating stoploss on exchange orders fails. The below is the default which is used if this is not configured in either strategy or configuration file. Syntax for Strategy: @@ -212,6 +220,7 @@ Syntax for Strategy: order_types = { "buy": "limit", "sell": "limit", + "emergencysell": "market", "stoploss": "market", "stoploss_on_exchange": False, "stoploss_on_exchange_interval": 60 @@ -224,6 +233,7 @@ Configuration: "order_types": { "buy": "limit", "sell": "limit", + "emergencysell": "market", "stoploss": "market", "stoploss_on_exchange": false, "stoploss_on_exchange_interval": 60 @@ -238,11 +248,13 @@ Configuration: !!! Note Stoploss on exchange interval is not mandatory. Do not change its value if you are unsure of what you are doing. For more information about how stoploss works please - read [the stoploss documentation](stoploss.md). + refer to [the stoploss documentation](stoploss.md). !!! Note - In case of stoploss on exchange if the stoploss is cancelled manually then - the bot would recreate one. + If `stoploss_on_exchange` is enabled and the stoploss is cancelled manually on the exchange, then the bot will create a new order. + +!!! Warning stoploss_on_exchange failures + If stoploss on exchange creation fails for some reason, then an "emergency sell" is initiated. By default, this will sell the asset using a market order. The order-type for the emergency-sell can be changed by setting the `emergencysell` value in the `order_types` dictionary - however this is not advised. ### Understand order_time_in_force diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 2f077edb7..cf292cacd 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -91,7 +91,8 @@ df.groupby("pair")["sell_reason"].value_counts() ### Load multiple configuration files -This option can be useful to inspect the results of passing in multiple configs +This option can be useful to inspect the results of passing in multiple configs. +This will also run through the whole Configuration initialization, so the configuration is completely initialized to be passed to other methods. ``` python import json @@ -101,7 +102,16 @@ from freqtrade.configuration import Configuration config = Configuration.from_files(["config1.json", "config2.json"]) # Show the config in memory -print(json.dumps(config, indent=1)) +print(json.dumps(config['original_config'], indent=2)) +``` + +For Interactive environments, have an additional configuration specifying `user_data_dir` and pass this in last, so you don't have to change directories while running the bot. +Best avoid relative paths, since this starts at the storage location of the jupyter notebook, unless the directory is changed. + +``` json +{ + "user_data_dir": "~/.freqtrade/" +} ``` ### Load exchange data to a pandas dataframe @@ -139,7 +149,7 @@ You can override strategy settings as demonstrated below. # Define some constants ticker_interval = "5m" # Name of the strategy class -strategy_name = 'TestStrategy' +strategy_name = 'SampleStrategy' # Path to user data user_data_dir = 'user_data' # Location of the strategy diff --git a/docs/data-download.md b/docs/data-download.md new file mode 100644 index 000000000..244acb153 --- /dev/null +++ b/docs/data-download.md @@ -0,0 +1,62 @@ +# Data Downloading + +## Getting data for backtesting and hyperopt + +To download data (candles / OHLCV) needed for backtesting and hyperoptimization use the `freqtrade download-data` command. + +If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes for the last 30 days. +Exchange and pairs will come from `config.json` (if specified using `-c/--config`). +Otherwise `--exchange` becomes mandatory. + +!!! Tip Updating existing data + If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. + Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. + +### Pairs file + +In alternative to the whitelist from `config.json`, a `pairs.json` file can be used. + +If you are using Binance for example: + +- create a directory `user_data/data/binance` and copy or create the `pairs.json` file in that directory. +- update the `pairs.json` file to contain the currency pairs you are interested in. + +```bash +mkdir -p user_data/data/binance +cp freqtrade/tests/testdata/pairs.json user_data/data/binance +``` + +The format of the `pairs.json` file is a simple json list. +Mixing different stake-currencies is allowed for this file, since it's only used for downloading. + +``` json +[ + "ETH/BTC", + "ETH/USDT", + "BTC/USDT", + "XRP/ETH" +] +``` + +### start download + +Then run: + +```bash +freqtrade download-data --exchange binance +``` + +This will download ticker data for all the currency pairs you defined in `pairs.json`. + +### Other Notes + +- To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. +- To change the exchange used to download the tickers, please use a different configuration file (you'll probably need to adjust ratelimits etc.) +- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. +- To download ticker data for only 10 days, use `--days 10` (defaults to 30 days). +- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. +- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. + +## Next step + +Great, you now have backtest data downloaded, so you can now start [backtesting](backtesting.md) your strategy. diff --git a/docs/deprecated.md b/docs/deprecated.md index ed70b1936..349d41a09 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -4,7 +4,7 @@ This page contains description of the command line arguments, configuration para and the bot features that were declared as DEPRECATED by the bot development team and are no longer supported. Please avoid their usage in your configuration. -## Deprecated +## Removed features ### the `--refresh-pairs-cached` command line option @@ -12,9 +12,7 @@ and are no longer supported. Please avoid their usage in your configuration. Since this leads to much confusion, and slows down backtesting (while not being part of backtesting) this has been singled out as a seperate freqtrade subcommand `freqtrade download-data`. -This command line option was deprecated in `2019.7-dev` and will be removed after the next release. - -## Removed features +This command line option was deprecated in 2019.7-dev (develop branch) and removed in 2019.9 (master branch). ### The **--dynamic-whitelist** command line option diff --git a/docs/developer.md b/docs/developer.md index 259bfafd8..627627b07 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -2,7 +2,7 @@ This page is intended for developers of FreqTrade, people who want to contribute to the FreqTrade codebase or documentation, or people who want to understand the source code of the application they're running. -All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel in [slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg) where you can ask questions. +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel in [slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) where you can ask questions. ## Documentation @@ -30,7 +30,7 @@ These are available from `conftest.py` and can be imported in any test module. A sample check looks as follows: ``` python -from freqtrade.tests.conftest import log_has, log_has_re +from tests.conftest import log_has, log_has_re def test_method_to_test(caplog): method_to_test() diff --git a/docs/hyperopt.md b/docs/hyperopt.md index c1bf56a3d..1ca371e3d 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -6,6 +6,9 @@ algorithms included in the `scikit-optimize` package to accomplish this. The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time. +Hyperopt requires historic data to be available, just as backtesting does. +To learn how to get data for the pairs and exchange you're interrested in, head over to the [Data Downloading](data-download.md) section of the documentation. + !!! Bug Hyperopt will crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133) @@ -372,35 +375,42 @@ Buy hyperspace params: 'rsi-enabled': True, 'trigger': 'bb_lower'} ROI table: -{ 0: 0.10674752302642071, - 21: 0.09158372701087236, - 78: 0.03634636907306948, +{ 0: 0.10674, + 21: 0.09158, + 78: 0.03634, 118: 0} ``` -This would translate to the following ROI table: +In order to use this best ROI table found by Hyperopt in backtesting and for live trades/dry-run, copy-paste it as the value of the `minimal_roi` attribute of your custom strategy: -``` python -minimal_roi = { - "118": 0, - "78": 0.0363, - "21": 0.0915, - "0": 0.106 +``` + # Minimal ROI designed for the strategy. + # This attribute will be overridden if the config file contains "minimal_roi" + minimal_roi = { + 0: 0.10674, + 21: 0.09158, + 78: 0.03634, + 118: 0 } ``` +As stated in the comment, you can also use it as the value of the `minimal_roi` setting in the configuration file. -If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps) with the values that can vary in the following ranges: +#### Default ROI Search Space -| # | minutes | ROI percentage | -|---|---|---| -| 1 | always 0 | 0.03...0.31 | -| 2 | 10...40 | 0.02...0.11 | -| 3 | 20...100 | 0.01...0.04 | -| 4 | 30...220 | always 0 | +If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the ticker_interval used. By default the values can vary in the following ranges (for some of the most used ticker intervals, values are rounded to 5 digits after the decimal point): -This structure of the ROI table is sufficient in most cases. Override the `roi_space()` method defining the ranges desired if you need components of the ROI tables to vary in other ranges. +| # step | 1m | | 5m | | 1h | | 1d | | +|---|---|---|---|---|---|---|---|---| +| 1 | 0 | 0.01161...0.11992 | 0 | 0.03...0.31 | 0 | 0.06883...0.71124 | 0 | 0.12178...1.25835 | +| 2 | 2...8 | 0.00774...0.04255 | 10...40 | 0.02...0.11 | 120...480 | 0.04589...0.25238 | 2880...11520 | 0.08118...0.44651 | +| 3 | 4...20 | 0.00387...0.01547 | 20...100 | 0.01...0.04 | 240...1200 | 0.02294...0.09177 | 5760...28800 | 0.04059...0.16237 | +| 4 | 6...44 | 0.0 | 30...220 | 0.0 | 360...2640 | 0.0 | 8640...63360 | 0.0 | -Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization in these methods if you need a different structure of the ROI table or other amount of rows (steps) in the ROI tables. +These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the ticker interval used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the ticker interval used. + +If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default. + +Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). A sample for these methods can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py). ### Understand Hyperopt Stoploss results @@ -417,12 +427,25 @@ Buy hyperspace params: 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'bb_lower'} -Stoploss: -0.37996664668703606 +Stoploss: -0.27996 ``` -If you are optimizing stoploss values, Freqtrade creates the 'stoploss' optimization hyperspace for you. By default, the stoploss values in that hyperspace can vary in the range -0.5...-0.02, which is sufficient in most cases. +In order to use this best stoploss value found by Hyperopt in backtesting and for live trades/dry-run, copy-paste it as the value of the `stoploss` attribute of your custom strategy: -Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. +``` + # Optimal stoploss designed for the strategy + # This attribute will be overridden if the config file contains "stoploss" + stoploss = -0.27996 +``` +As stated in the comment, you can also use it as the value of the `stoploss` setting in the configuration file. + +#### Default Stoploss Search Space + +If you are optimizing stoploss values, Freqtrade creates the 'stoploss' optimization hyperspace for you. By default, the stoploss values in that hyperspace can vary in the range -0.35...-0.02, which is sufficient in most cases. + +If you have the `stoploss_space()` method in your custom hyperopt file, remove it in order to utilize Stoploss hyperoptimization space generated by Freqtrade by default. + +Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py). ### Validate backtesting results diff --git a/docs/index.md b/docs/index.md index 63d6be75e..206d635c6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -64,7 +64,7 @@ To run this bot we recommend you a cloud instance with a minimum of: Help / Slack For any questions not covered by the documentation or for further information about the bot, we encourage you to join our Slack channel. -Click [here](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg) to join Slack channel. +Click [here](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) to join Slack channel. ## Ready to try? diff --git a/docs/installation.md b/docs/installation.md index f15cc356c..081d7e0cf 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -99,8 +99,8 @@ sudo apt-get install build-essential git Before installing FreqTrade on a Raspberry Pi running the official Raspbian Image, make sure you have at least Python 3.6 installed. The default image only provides Python 3.5. Probably the easiest way to get a recent version of python is [miniconda](https://repo.continuum.io/miniconda/). -The following assumes that miniconda3 is installed and available in your environment. Last miniconda3 installation file use python 3.4, we will update to python 3.6 on this installation. -It's recommended to use (mini)conda for this as installation/compilation of `numpy`, `scipy` and `pandas` takes a long time. +The following assumes that miniconda3 is installed and available in your environment. Since the last miniconda3 installation file uses python 3.4, we will update to python 3.6 on this installation. +It's recommended to use (mini)conda for this as installation/compilation of `numpy` and `pandas` takes a long time. Additional package to install on your Raspbian, `libffi-dev` required by cryptography (from python-telegram-bot). @@ -109,13 +109,17 @@ conda config --add channels rpi conda install python=3.6 conda create -n freqtrade python=3.6 conda activate freqtrade -conda install scipy pandas numpy +conda install pandas numpy sudo apt install libffi-dev python3 -m pip install -r requirements-common.txt python3 -m pip install -e . ``` +!!! Note + This does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. + We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerful machine. + ### Common #### 1. Install TA-Lib @@ -175,7 +179,6 @@ cp config.json.example config.json ``` bash python3 -m pip install --upgrade pip -python3 -m pip install -r requirements.txt python3 -m pip install -e . ``` diff --git a/docs/plotting.md b/docs/plotting.md index 5a1e9757a..4deb6db12 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -2,9 +2,9 @@ This page explains how to plot prices, indicators and profits. -## Installation +## Installation / Setup -Plotting scripts use Plotly library. Install/upgrade it with: +Plotting modules use the Plotly library. You can install / upgrade this by running the following command: ``` bash pip install -U -r requirements-plot.txt @@ -12,90 +12,172 @@ pip install -U -r requirements-plot.txt ## Plot price and indicators -Usage for the price plotter: +The `freqtrade plot-dataframe` subcommand shows an interactive graph with three subplots: + +* Main plot with candlestics and indicators following price (sma/ema) +* Volume bars +* Additional indicators as specified by `--indicators2` + +![plot-dataframe](assets/plot-dataframe.png) + +Possible arguments: + +``` +usage: freqtrade plot-dataframe [-h] [-p PAIRS [PAIRS ...]] + [--indicators1 INDICATORS1 [INDICATORS1 ...]] + [--indicators2 INDICATORS2 [INDICATORS2 ...]] + [--plot-limit INT] [--db-url PATH] + [--trade-source {DB,file}] [--export EXPORT] + [--export-filename PATH] + [--timerange TIMERANGE] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space- + separated. + --indicators1 INDICATORS1 [INDICATORS1 ...] + Set indicators from your strategy you want in the + first row of the graph. Space-separated list. Example: + `ema3 ema5`. Default: `['sma', 'ema3', 'ema5']`. + --indicators2 INDICATORS2 [INDICATORS2 ...] + Set indicators from your strategy you want in the + third row of the graph. Space-separated list. Example: + `fastd fastk`. Default: `['macd', 'macdsignal']`. + --plot-limit INT Specify tick limit for plotting. Notice: too high + values cause huge files. Default: 750. + --db-url PATH Override trades database URL, this is useful in custom + deployments (default: `sqlite:///tradesv3.sqlite` for + Live Run mode, `sqlite://` for Dry Run). + --trade-source {DB,file} + Specify the source for trades (Can be DB or file + (backtest file)) Default: file + --export EXPORT Export backtest results, argument are: trades. + Example: `--export=trades` + --export-filename PATH + Save backtest results to the file with this filename + (default: `user_data/backtest_results/backtest- + result.json`). Requires `--export` to be set as well. + Example: `--export-filename=user_data/backtest_results + /backtest_today.json` + --timerange TIMERANGE + Specify what timerange of data to use. -``` bash -python3 script/plot_dataframe.py [-h] [-p pairs] ``` -Example +Example: ``` bash -python3 scripts/plot_dataframe.py -p BTC/ETH +freqtrade plot-dataframe -p BTC/ETH ``` -The `-p` pairs argument can be used to specify pairs you would like to plot. +The `-p/--pairs` argument can be used to specify pairs you would like to plot. + +!!! Note + The `freqtrade plot-dataframe` subcommand generates one plot-file per pair. Specify custom indicators. Use `--indicators1` for the main plot and `--indicators2` for the subplot below (if values are in a different range than prices). +!!! tip + You will almost certainly want to specify a custom strategy! This can be done by adding `-s Classname` / `--strategy ClassName` to the command. + ``` bash -python3 scripts/plot_dataframe.py -p BTC/ETH --indicators1 sma,ema --indicators2 macd +freqtrade --strategy AwesomeStrategy plot-dataframe -p BTC/ETH --indicators1 sma ema --indicators2 macd ``` -### Advanced use +### Further usage examples -To plot multiple pairs, separate them with a comma: +To plot multiple pairs, separate them with a space: ``` bash -python3 scripts/plot_dataframe.py -p BTC/ETH,XRP/ETH +freqtrade --strategy AwesomeStrategy plot-dataframe -p BTC/ETH XRP/ETH ``` -To plot a timerange (to zoom in): +To plot a timerange (to zoom in) ``` bash -python3 scripts/plot_dataframe.py -p BTC/ETH --timerange=20180801-20180805 +freqtrade --strategy AwesomeStrategy plot-dataframe -p BTC/ETH --timerange=20180801-20180805 ``` -To plot trades stored in a database use `--db-url` argument: +To plot trades stored in a database use `--db-url` in combination with `--trade-source DB`: ``` bash -python3 scripts/plot_dataframe.py --db-url sqlite:///tradesv3.dry_run.sqlite -p BTC/ETH --trade-source DB +freqtrade --strategy AwesomeStrategy plot-dataframe --db-url sqlite:///tradesv3.dry_run.sqlite -p BTC/ETH --trade-source DB ``` To plot trades from a backtesting result, use `--export-filename ` ``` bash -python3 scripts/plot_dataframe.py --export-filename user_data/backtest_results/backtest-result.json -p BTC/ETH -``` - -To plot a custom strategy the strategy should have first be backtested. -The results may then be plotted with the -s argument: - -``` bash -python3 scripts/plot_dataframe.py -s Strategy_Name -p BTC/ETH --datadir user_data/data// +freqtrade --strategy AwesomeStrategy plot-dataframe --export-filename user_data/backtest_results/backtest-result.json -p BTC/ETH ``` ## Plot profit -The profit plotter shows a picture with three plots: +![plot-profit](assets/plot-profit.png) + +The `freqtrade plot-profit` subcommand shows an interactive graph with three plots: 1) Average closing price for all pairs 2) The summarized profit made by backtesting. - Note that this is not the real-world profit, but - more of an estimate. -3) Each pair individually profit + Note that this is not the real-world profit, but more of an estimate. +3) Profit for each individual pair -The first graph is good to get a grip of how the overall market -progresses. +The first graph is good to get a grip of how the overall market progresses. -The second graph will show how your algorithm works or doesn't. -Perhaps you want an algorithm that steadily makes small profits, -or one that acts less seldom, but makes big swings. +The second graph will show if your algorithm works or doesn't. +Perhaps you want an algorithm that steadily makes small profits, or one that acts less often, but makes big swings. -The third graph can be useful to spot outliers, events in pairs -that makes profit spikes. +The third graph can be useful to spot outliers, events in pairs that cause profit spikes. -Usage for the profit plotter: +Possible options for the `freqtrade plot-profit` subcommand: + +``` +usage: freqtrade plot-profit [-h] [-p PAIRS [PAIRS ...]] + [--timerange TIMERANGE] [--export EXPORT] + [--export-filename PATH] [--db-url PATH] + [--trade-source {DB,file}] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space- + separated. + --timerange TIMERANGE + Specify what timerange of data to use. + --export EXPORT Export backtest results, argument are: trades. + Example: `--export=trades` + --export-filename PATH + Save backtest results to the file with this filename + (default: `user_data/backtest_results/backtest- + result.json`). Requires `--export` to be set as well. + Example: `--export-filename=user_data/backtest_results + /backtest_today.json` + --db-url PATH Override trades database URL, this is useful in custom + deployments (default: `sqlite:///tradesv3.sqlite` for + Live Run mode, `sqlite://` for Dry Run). + --trade-source {DB,file} + Specify the source for trades (Can be DB or file + (backtest file)) Default: file -``` bash -python3 script/plot_profit.py [-h] [-p pair] [--datadir directory] [--ticker_interval num] ``` -The `-p` pair argument, can be used to plot a single pair +The `-p/--pairs` argument, can be used to limit the pairs that are considered for this calculation. -Example +Examples: + +Use custom backtest-export file ``` bash -python3 scripts/plot_profit.py --datadir ../freqtrade/freqtrade/tests/testdata-20171221/ -p LTC/BTC +freqtrade plot-profit -p LTC/BTC --export-filename user_data/backtest_results/backtest-result-Strategy005.json +``` + +Use custom database + +``` bash +freqtrade plot-profit -p LTC/BTC --db-url sqlite:///tradesv3.sqlite --trade-source DB +``` + +``` bash +freqtrade plot-profit --datadir user_data/data/binance_save/ -p LTC/BTC ``` diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index ce76d52e5..28ebc5916 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1 +1 @@ -mkdocs-material==4.4.0 \ No newline at end of file +mkdocs-material==4.4.2 \ No newline at end of file diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 9e32ded18..b927e5aad 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -24,7 +24,7 @@ strategy file will be updated on Github. Put your custom strategy file into the directory `user_data/strategies`. Best copy the test-strategy and modify this copy to avoid having bot-updates override your changes. -`cp user_data/strategies/test_strategy.py user_data/strategies/awesome-strategy.py` +`cp user_data/strategies/sample_strategy.py user_data/strategies/awesome-strategy.py` ### Anatomy of a strategy @@ -36,14 +36,19 @@ A strategy file contains all the information needed to build a good strategy: - Minimal ROI recommended - Stoploss strongly recommended -The bot also include a sample strategy called `TestStrategy` you can update: `user_data/strategies/test_strategy.py`. -You can test it with the parameter: `--strategy TestStrategy` +The bot also include a sample strategy called `SampleStrategy` you can update: `user_data/strategies/sample_strategy.py`. +You can test it with the parameter: `--strategy SampleStrategy` + +Additionally, there is an attribute called `INTERFACE_VERSION`, which defines the version of the strategy interface the bot should use. +The current version is 2 - which is also the default when it's not set explicitly in the strategy. + +Future versions will require this to be set. ```bash freqtrade --strategy AwesomeStrategy ``` -**For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +**For the following section we will use the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/sample_strategy.py) file as reference.** !!! Note Strategies and Backtesting @@ -109,9 +114,8 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame return dataframe ``` - !!! Note "Want more indicator examples?" - Look into the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py).
+ Look into the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/sample_strategy.py). Then uncomment indicators you need. ### Buy signal rules @@ -122,7 +126,7 @@ It's important to always return the dataframe without removing/modifying the col This will method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action". -Sample from `user_data/strategies/test_strategy.py`: +Sample from `user_data/strategies/sample_strategy.py`: ```python def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -152,7 +156,7 @@ It's important to always return the dataframe without removing/modifying the col This will method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action". -Sample from `user_data/strategies/test_strategy.py`: +Sample from `user_data/strategies/sample_strategy.py`: ```python def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -220,7 +224,7 @@ This would signify a stoploss of -10%. For the full documentation on stoploss features, look at the dedicated [stoploss page](stoploss.md). -If your exchange supports it, it's recommended to also set `"stoploss_on_exchange"` in the order dict, so your stoploss is on the exchange and cannot be missed for network-problems (or other problems). +If your exchange supports it, it's recommended to also set `"stoploss_on_exchange"` in the order_types dictionary, so your stoploss is on the exchange and cannot be missed due to network problems, high load or other reasons. For more information on order_types please look [here](configuration.md#understand-order_types). @@ -407,7 +411,7 @@ To get additional Ideas for strategies, head over to our [strategy repository](h Feel free to use any of them as inspiration for your own strategies. We're happy to accept Pull Requests containing new Strategies to that repo. -We also got a *strategy-sharing* channel in our [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg) which is a great place to get and/or share ideas. +We also got a *strategy-sharing* channel in our [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) which is a great place to get and/or share ideas. ## Next step diff --git a/environment.yml b/environment.yml index cd3350fd5..4e8c1efcc 100644 --- a/environment.yml +++ b/environment.yml @@ -9,25 +9,26 @@ dependencies: - wheel - numpy - pandas - - scipy - SQLAlchemy - - scikit-learn - arrow - requests - urllib3 - wrapt - - joblib - jsonschema - tabulate - python-rapidjson - - filelock - flask - python-dotenv - cachetools - - scikit-optimize - python-telegram-bot # Optional for plotting - plotly + # Optional for hyperopt + - scipy + - scikit-optimize + - scikit-learn + - filelock + - joblib # Optional for development - flake8 - pytest diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index bd44b9eb7..6ec7bce21 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,16 @@ """ FreqTrade bot """ -__version__ = '2019.8-1' +__version__ = '2019.9' + +if __version__ == 'develop': + + try: + import subprocess + __version__ = 'develop-' + subprocess.check_output( + ['git', 'log', '--format="%h"', '-n 1'], + stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') + except Exception: + # git not available, ignore + pass class DependencyException(Exception): @@ -11,7 +22,7 @@ class DependencyException(Exception): class OperationalException(Exception): """ - Requires manual intervention. + Requires manual intervention and will usually stop the bot. This happens when an exchange returns an unexpected error during runtime or given configuration is invalid. """ diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 2c76afa8f..6e2ecea2e 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -2,10 +2,11 @@ This module contains the argument manager class """ import argparse -from typing import List, Optional +from pathlib import Path +from typing import Any, Dict, List, Optional -from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS from freqtrade import constants +from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"] @@ -14,7 +15,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"] ARGS_MAIN = ARGS_COMMON + ARGS_STRATEGY + ["db_url", "sd_notify"] ARGS_COMMON_OPTIMIZE = ["ticker_interval", "timerange", - "max_open_trades", "stake_amount", "refresh_pairs"] + "max_open_trades", "stake_amount"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", "strategy_list", "export", "exportfilename"] @@ -34,33 +35,29 @@ ARGS_CREATE_USERDIR = ["user_data_dir"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] -ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + - ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", - "trade_source", "export", "exportfilename", "timerange", - "refresh_pairs"]) +ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", + "trade_source", "export", "exportfilename", "timerange", "ticker_interval"] -ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY + - ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"]) +ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", + "trade_source", "ticker_interval"] -NO_CONF_REQURIED = ["start_download_data"] +NO_CONF_REQURIED = ["create-userdir", "download-data", "plot-dataframe", "plot-profit"] -class Arguments(object): +class Arguments: """ Arguments Class. Manage the arguments received by the cli """ - def __init__(self, args: Optional[List[str]], description: str, - no_default_config: bool = False) -> None: + def __init__(self, args: Optional[List[str]]) -> None: self.args = args self._parsed_arg: Optional[argparse.Namespace] = None - self.parser = argparse.ArgumentParser(description=description) - self._no_default_config = no_default_config + self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') def _load_args(self) -> None: self._build_args(optionlist=ARGS_MAIN) self._build_subcommands() - def get_parsed_arg(self) -> argparse.Namespace: + def get_parsed_arg(self) -> Dict[str, Any]: """ Return the list of arguments :return: List[str] List of arguments @@ -69,7 +66,7 @@ class Arguments(object): self._load_args() self._parsed_arg = self._parse_args() - return self._parsed_arg + return vars(self._parsed_arg) def _parse_args(self) -> argparse.Namespace: """ @@ -77,12 +74,13 @@ class Arguments(object): """ parsed_arg = self.parser.parse_args(self.args) + # When no config is provided, but a config exists, use that configuration! + # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) # Allow no-config for certain commands (like downloading / plotting) - if (not self._no_default_config and parsed_arg.config is None - and not (hasattr(parsed_arg, 'func') - and parsed_arg.func.__name__ in NO_CONF_REQURIED)): + if (parsed_arg.config is None and ((Path.cwd() / constants.DEFAULT_CONFIG).is_file() or + not ('subparser' in parsed_arg and parsed_arg.subparser in NO_CONF_REQURIED))): parsed_arg.config = [constants.DEFAULT_CONFIG] return parsed_arg @@ -119,6 +117,7 @@ class Arguments(object): hyperopt_cmd.set_defaults(func=start_hyperopt) self._build_args(optionlist=ARGS_HYPEROPT, parser=hyperopt_cmd) + # add create-userdir subcommand create_userdir_cmd = subparsers.add_parser('create-userdir', help="Create user-data directory.") create_userdir_cmd.set_defaults(func=start_create_userdir) @@ -139,3 +138,20 @@ class Arguments(object): ) download_data_cmd.set_defaults(func=start_download_data) self._build_args(optionlist=ARGS_DOWNLOAD_DATA, parser=download_data_cmd) + + # Add Plotting subcommand + from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit + plot_dataframe_cmd = subparsers.add_parser( + 'plot-dataframe', + help='Plot candles with indicators.' + ) + plot_dataframe_cmd.set_defaults(func=start_plot_dataframe) + self._build_args(optionlist=ARGS_PLOT_DATAFRAME, parser=plot_dataframe_cmd) + + # Plot profit + plot_profit_cmd = subparsers.add_parser( + 'plot-profit', + help='Generate plot showing profits.' + ) + plot_profit_cmd.set_defaults(func=start_plot_profit) + self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd) diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index 70f4cfa33..19c377732 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -5,6 +5,7 @@ from freqtrade import OperationalException from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, is_exchange_available, is_exchange_bad, is_exchange_officially_supported) +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -19,9 +20,21 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: raises an exception if the exchange if not supported by ccxt and thus is not known for the Freqtrade at all. """ + + if config['runmode'] in [RunMode.PLOT] and not config.get('exchange', {}).get('name'): + # Skip checking exchange in plot mode, since it requires no exchange + return True logger.info("Checking exchange...") exchange = config.get('exchange', {}).get('name').lower() + if not exchange: + raise OperationalException( + f'This command requires a configured exchange. You should either use ' + f'`--exchange ` or specify a configuration file via `--config`.\n' + f'The following exchanges are supported by ccxt: ' + f'{", ".join(available_exchanges())}' + ) + if not is_exchange_available(exchange): raise OperationalException( f'Exchange "{exchange}" is not supported by ccxt ' diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index d7e4e61b1..cb07dbdba 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -107,13 +107,6 @@ AVAILABLE_CLI_OPTIONS = { help='Specify stake_amount.', type=float, ), - "refresh_pairs": Arg( - '-r', '--refresh-pairs-cached', - help='Refresh the pairs files in tests/testdata with the latest data from the ' - 'exchange. Use it if you want to run your optimization commands with ' - 'up-to-date data.', - action='store_true', - ), # Backtesting "position_stacking": Arg( '--eps', '--enable-position-stacking', @@ -292,14 +285,16 @@ AVAILABLE_CLI_OPTIONS = { "indicators1": Arg( '--indicators1', help='Set indicators from your strategy you want in the first row of the graph. ' - 'Comma-separated list. Example: `ema3,ema5`. Default: `%(default)s`.', - default='sma,ema3,ema5', + 'Space-separated list. Example: `ema3 ema5`. Default: `%(default)s`.', + default=['sma', 'ema3', 'ema5'], + nargs='+', ), "indicators2": Arg( '--indicators2', help='Set indicators from your strategy you want in the third row of the graph. ' - 'Comma-separated list. Example: `fastd,fastk`. Default: `%(default)s`.', - default='macd,macdsignal', + 'Space-separated list. Example: `fastd fastk`. Default: `%(default)s`.', + default=['macd', 'macdsignal'], + nargs='+', ), "plot_limit": Arg( '--plot-limit', diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index b1bd3ef1c..764593d0f 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -3,7 +3,6 @@ This module contains the configuration class """ import logging import warnings -from argparse import Namespace from copy import deepcopy from pathlib import Path from typing import Any, Callable, Dict, List, Optional @@ -22,13 +21,13 @@ from freqtrade.state import RunMode logger = logging.getLogger(__name__) -class Configuration(object): +class Configuration: """ Class to read and init the bot configuration Reuse this class for the bot, backtesting, hyperopt and every script that required configuration """ - def __init__(self, args: Namespace, runmode: RunMode = None) -> None: + def __init__(self, args: Dict[str, Any], runmode: RunMode = None) -> None: self.args = args self.config: Optional[Dict[str, Any]] = None self.runmode = runmode @@ -50,9 +49,16 @@ class Configuration(object): and merging their contents. Files are loaded in sequence, parameters in later configuration files override the same parameter from an earlier file (last definition wins). + Runs through the whole Configuration initialization, so all expected config entries + are available to interactive environments. :param files: List of file paths :return: configuration dictionary """ + c = Configuration({"config": files}, RunMode.OTHER) + return c.get_config() + + def load_from_files(self, files: List[str]) -> Dict[str, Any]: + # Keep this method as staticmethod, so it can be used from interactive environments config: Dict[str, Any] = {} @@ -82,7 +88,10 @@ class Configuration(object): :return: Configuration dictionary """ # Load all configs - config: Dict[str, Any] = Configuration.from_files(self.args.config) + config: Dict[str, Any] = self.load_from_files(self.args["config"]) + + # Keep a copy of the original configuration file + config['original_config'] = deepcopy(config) self._process_common_options(config) @@ -107,13 +116,10 @@ class Configuration(object): the -v/--verbose, --logfile options """ # Log level - if 'verbosity' in self.args and self.args.verbosity: - config.update({'verbosity': self.args.verbosity}) - else: - config.update({'verbosity': 0}) + config.update({'verbosity': self.args.get("verbosity", 0)}) - if 'logfile' in self.args and self.args.logfile: - config.update({'logfile': self.args.logfile}) + if 'logfile' in self.args and self.args["logfile"]: + config.update({'logfile': self.args["logfile"]}) setup_logging(config) @@ -122,15 +128,15 @@ class Configuration(object): self._process_logging_options(config) # Set strategy if not specified in config and or if it's non default - if self.args.strategy != constants.DEFAULT_STRATEGY or not config.get('strategy'): - config.update({'strategy': self.args.strategy}) + if self.args.get("strategy") != constants.DEFAULT_STRATEGY or not config.get('strategy'): + config.update({'strategy': self.args.get("strategy")}) self._args_to_config(config, argname='strategy_path', logstring='Using additional Strategy lookup path: {}') - if ('db_url' in self.args and self.args.db_url and - self.args.db_url != constants.DEFAULT_DB_PROD_URL): - config.update({'db_url': self.args.db_url}) + if ('db_url' in self.args and self.args["db_url"] and + self.args["db_url"] != constants.DEFAULT_DB_PROD_URL): + config.update({'db_url': self.args["db_url"]}) logger.info('Parameter --db-url detected ...') if config.get('dry_run', False): @@ -153,7 +159,7 @@ class Configuration(object): config['max_open_trades'] = float('inf') # Support for sd_notify - if 'sd_notify' in self.args and self.args.sd_notify: + if 'sd_notify' in self.args and self.args["sd_notify"]: config['internals'].update({'sd_notify': True}) def _process_datadir_options(self, config: Dict[str, Any]) -> None: @@ -162,12 +168,12 @@ class Configuration(object): --user-data, --datadir """ # Check exchange parameter here - otherwise `datadir` might be wrong. - if "exchange" in self.args and self.args.exchange: - config['exchange']['name'] = self.args.exchange + if "exchange" in self.args and self.args["exchange"]: + config['exchange']['name'] = self.args["exchange"] logger.info(f"Using exchange {config['exchange']['name']}") - if 'user_data_dir' in self.args and self.args.user_data_dir: - config.update({'user_data_dir': self.args.user_data_dir}) + if 'user_data_dir' in self.args and self.args["user_data_dir"]: + config.update({'user_data_dir': self.args["user_data_dir"]}) elif 'user_data_dir' not in config: # Default to cwd/user_data (legacy option ...) config.update({'user_data_dir': str(Path.cwd() / "user_data")}) @@ -176,10 +182,7 @@ class Configuration(object): config['user_data_dir'] = create_userdata_dir(config['user_data_dir'], create_dir=False) logger.info('Using user-data directory: %s ...', config['user_data_dir']) - if 'datadir' in self.args and self.args.datadir: - config.update({'datadir': create_datadir(config, self.args.datadir)}) - else: - config.update({'datadir': create_datadir(config, None)}) + config.update({'datadir': create_datadir(config, self.args.get("datadir", None))}) logger.info('Using data directory: %s ...', config.get('datadir')) def _process_optimize_options(self, config: Dict[str, Any]) -> None: @@ -192,12 +195,12 @@ class Configuration(object): self._args_to_config(config, argname='position_stacking', logstring='Parameter --enable-position-stacking detected ...') - if 'use_max_market_positions' in self.args and not self.args.use_max_market_positions: + if 'use_max_market_positions' in self.args and not self.args["use_max_market_positions"]: config.update({'use_max_market_positions': False}) logger.info('Parameter --disable-max-market-positions detected ...') logger.info('max_open_trades set to unlimited ...') - elif 'max_open_trades' in self.args and self.args.max_open_trades: - config.update({'max_open_trades': self.args.max_open_trades}) + elif 'max_open_trades' in self.args and self.args["max_open_trades"]: + config.update({'max_open_trades': self.args["max_open_trades"]}) logger.info('Parameter --max_open_trades detected, ' 'overriding max_open_trades to: %s ...', config.get('max_open_trades')) else: @@ -212,12 +215,8 @@ class Configuration(object): self._process_datadir_options(config) - self._args_to_config(config, argname='refresh_pairs', - logstring='Parameter -r/--refresh-pairs-cached detected ...', - deprecated_msg='-r/--refresh-pairs-cached will be removed soon.') - self._args_to_config(config, argname='strategy_list', - logstring='Using strategy list of {} Strategies', logfun=len) + logstring='Using strategy list of {} strategies', logfun=len) self._args_to_config(config, argname='ticker_interval', logstring='Overriding ticker interval with Command line argument') @@ -229,16 +228,16 @@ class Configuration(object): logstring='Storing backtest results to {} ...') # Edge section: - if 'stoploss_range' in self.args and self.args.stoploss_range: - txt_range = eval(self.args.stoploss_range) + if 'stoploss_range' in self.args and self.args["stoploss_range"]: + txt_range = eval(self.args["stoploss_range"]) config['edge'].update({'stoploss_range_min': txt_range[0]}) config['edge'].update({'stoploss_range_max': txt_range[1]}) config['edge'].update({'stoploss_range_step': txt_range[2]}) - logger.info('Parameter --stoplosses detected: %s ...', self.args.stoploss_range) + logger.info('Parameter --stoplosses detected: %s ...', self.args["stoploss_range"]) # Hyperopt section self._args_to_config(config, argname='hyperopt', - logstring='Using Hyperopt file {}') + logstring='Using Hyperopt class name: {}') self._args_to_config(config, argname='hyperopt_path', logstring='Using additional Hyperopt lookup path: {}') @@ -254,7 +253,7 @@ class Configuration(object): self._args_to_config(config, argname='print_all', logstring='Parameter --print-all detected ...') - if 'print_colorized' in self.args and not self.args.print_colorized: + if 'print_colorized' in self.args and not self.args["print_colorized"]: logger.info('Parameter --no-color detected ...') config.update({'print_colorized': False}) else: @@ -276,7 +275,7 @@ class Configuration(object): logstring='Hyperopt continue: {}') self._args_to_config(config, argname='hyperopt_loss', - logstring='Using loss function: {}') + logstring='Using Hyperopt loss class name: {}') def _process_plot_options(self, config: Dict[str, Any]) -> None: @@ -324,9 +323,9 @@ class Configuration(object): sample: logfun=len (prints the length of the found configuration instead of the content) """ - if argname in self.args and getattr(self.args, argname): + if argname in self.args and self.args[argname]: - config.update({argname: getattr(self.args, argname)}) + config.update({argname: self.args[argname]}) if logfun: logger.info(logstring.format(logfun(config[argname]))) else: @@ -346,8 +345,8 @@ class Configuration(object): if "pairs" in config: return - if "pairs_file" in self.args and self.args.pairs_file: - pairs_file = Path(self.args.pairs_file) + if "pairs_file" in self.args and self.args["pairs_file"]: + pairs_file = Path(self.args["pairs_file"]) logger.info(f'Reading pairs file "{pairs_file}".') # Download pairs from the pairs file if no config is specified # or if pairs file is specified explicitely @@ -358,7 +357,7 @@ class Configuration(object): config['pairs'].sort() return - if "config" in self.args and self.args.config: + if "config" in self.args and self.args["config"]: logger.info("Using pairlist from configuration.") config['pairs'] = config.get('exchange', {}).get('pair_whitelist') else: diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index f980b71ea..ec6eb75e6 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -7,7 +7,7 @@ from typing import Optional import arrow -class TimeRange(): +class TimeRange: """ object defining timerange inputs. [start/stop]type defines if [start/stop]ts shall be used. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 05ee99c1b..abf43b24d 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -22,6 +22,7 @@ ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList'] DRY_RUN_WALLET = 999.9 +MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons TICKER_INTERVALS = [ '1m', '3m', '5m', '15m', '30m', @@ -121,6 +122,7 @@ CONF_SCHEMA = { 'properties': { 'buy': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'sell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, + 'emergencysell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'stoploss_on_exchange': {'type': 'boolean'}, 'stoploss_on_exchange_interval': {'type': 'number'} diff --git a/freqtrade/data/__init__.py b/freqtrade/data/__init__.py index 0a31d095c..0e7eea0d0 100644 --- a/freqtrade/data/__init__.py +++ b/freqtrade/data/__init__.py @@ -2,7 +2,7 @@ Module to handle data operations for freqtrade """ -# limit what's imported when using `from freqtrad.data import *`` +# limit what's imported when using `from freqtrade.data import *` __all__ = [ 'converter' ] diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 36d8aedbb..b03bdb74d 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -112,16 +112,16 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: return trades -def load_trades(config) -> pd.DataFrame: +def load_trades(source: str, db_url: str, exportfilename: str) -> pd.DataFrame: """ Based on configuration option "trade_source": * loads data from DB (using `db_url`) * loads data from backtestfile (using `exportfilename`) """ - if config["trade_source"] == "DB": - return load_trades_from_db(config["db_url"]) - elif config["trade_source"] == "file": - return load_backtest_data(Path(config["exportfilename"])) + if source == "DB": + return load_trades_from_db(db_url) + elif source == "file": + return load_backtest_data(Path(exportfilename)) def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame: @@ -157,7 +157,8 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str) -> :param trades: DataFrame containing trades (requires columns close_time and profitperc) :return: Returns df with one additional column, col_name, containing the cumulative profit. """ - df[col_name] = trades.set_index('close_time')['profitperc'].cumsum() + # Use groupby/sum().cumsum() to avoid errors when multiple trades sold at the same candle. + df[col_name] = trades.groupby('close_time')['profitperc'].sum().cumsum() # Set first value to 0 df.loc[df.iloc[0].name, col_name] = 0 # FFill to get continuous diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 5b71c21a8..7d5e4540b 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -17,7 +17,7 @@ from freqtrade.state import RunMode logger = logging.getLogger(__name__) -class DataProvider(): +class DataProvider: def __init__(self, config: dict, exchange: Exchange) -> None: self._config = config @@ -65,9 +65,7 @@ class DataProvider(): """ return load_pair_history(pair=pair, ticker_interval=ticker_interval or self._config['ticker_interval'], - refresh_pairs=False, - datadir=Path(self._config['datadir']) if self._config.get( - 'datadir') else None + datadir=Path(self._config['datadir']) ) def get_pair_dataframe(self, pair: str, ticker_interval: str = None) -> DataFrame: diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 007357d9a..865289f38 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -57,11 +57,11 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: return tickerlist[start_index:stop_index] -def load_tickerdata_file(datadir: Optional[Path], pair: str, ticker_interval: str, +def load_tickerdata_file(datadir: Path, pair: str, ticker_interval: str, timerange: Optional[TimeRange] = None) -> Optional[list]: """ Load a pair from file, either .json.gz or .json - :return: tickerlist or None if unsuccesful + :return: tickerlist or None if unsuccessful """ filename = pair_data_filename(datadir, pair, ticker_interval) pairdata = misc.file_load_json(filename) @@ -73,7 +73,7 @@ def load_tickerdata_file(datadir: Optional[Path], pair: str, ticker_interval: st return pairdata -def store_tickerdata_file(datadir: Optional[Path], pair: str, +def store_tickerdata_file(datadir: Path, pair: str, ticker_interval: str, data: list, is_zip: bool = False): """ Stores tickerdata to file @@ -84,7 +84,7 @@ def store_tickerdata_file(datadir: Optional[Path], pair: str, def load_pair_history(pair: str, ticker_interval: str, - datadir: Optional[Path], + datadir: Path, timerange: TimeRange = TimeRange(None, None, 0, 0), refresh_pairs: bool = False, exchange: Optional[Exchange] = None, @@ -129,62 +129,47 @@ def load_pair_history(pair: str, else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' - 'Use --refresh-pairs-cached option or `freqtrade download-data` ' - 'script to download the data' + 'Use `freqtrade download-data` to download the data' ) return None -def load_data(datadir: Optional[Path], +def load_data(datadir: Path, ticker_interval: str, pairs: List[str], refresh_pairs: bool = False, exchange: Optional[Exchange] = None, timerange: TimeRange = TimeRange(None, None, 0, 0), fill_up_missing: bool = True, - live: bool = False ) -> Dict[str, DataFrame]: """ - Loads ticker history data for a list of pairs the given parameters + Loads ticker history data for a list of pairs :return: dict(:) + TODO: refresh_pairs is still used by edge to keep the data uptodate. + This should be replaced in the future. Instead, writing the current candles to disk + from dataprovider should be implemented, as this would avoid loading ohlcv data twice. + exchange and refresh_pairs are then not needed here nor in load_pair_history. """ result: Dict[str, DataFrame] = {} - if live: - if exchange: - logger.info('Live: Downloading data for all defined pairs ...') - exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs]) - result = {key[0]: value for key, value in exchange._klines.items() if value is not None} - else: - raise OperationalException( - "Exchange needs to be initialized when using live data." - ) - else: - logger.info('Using local backtesting data ...') - for pair in pairs: - hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, - datadir=datadir, timerange=timerange, - refresh_pairs=refresh_pairs, - exchange=exchange, - fill_up_missing=fill_up_missing) - if hist is not None: - result[pair] = hist + for pair in pairs: + hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, + datadir=datadir, timerange=timerange, + refresh_pairs=refresh_pairs, + exchange=exchange, + fill_up_missing=fill_up_missing) + if hist is not None: + result[pair] = hist return result -def make_testdata_path(datadir: Optional[Path]) -> Path: - """Return the path where testdata files are stored""" - return datadir or (Path(__file__).parent.parent / "tests" / "testdata").resolve() - - -def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) -> Path: - path = make_testdata_path(datadir) +def pair_data_filename(datadir: Path, pair: str, ticker_interval: str) -> Path: pair_s = pair.replace("/", "_") - filename = path.joinpath(f'{pair_s}-{ticker_interval}.json') + filename = datadir.joinpath(f'{pair_s}-{ticker_interval}.json') return filename -def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_interval: str, +def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: """ @@ -224,7 +209,7 @@ def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_int return (data, since_ms) -def download_pair_history(datadir: Optional[Path], +def download_pair_history(datadir: Path, exchange: Optional[Exchange], pair: str, ticker_interval: str = '5m', @@ -280,6 +265,35 @@ def download_pair_history(datadir: Optional[Path], return False +def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], + dl_path: Path, timerange: TimeRange, + erase=False) -> List[str]: + """ + Refresh stored ohlcv data for backtesting and hyperopt operations. + Used by freqtrade download-data + :return: Pairs not available + """ + pairs_not_available = [] + for pair in pairs: + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for ticker_interval in timeframes: + + dl_file = pair_data_filename(dl_path, pair, ticker_interval) + if erase and dl_file.exists(): + logger.info( + f'Deleting existing data for pair {pair}, interval {ticker_interval}.') + dl_file.unlink() + + logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, ticker_interval=str(ticker_interval), + timerange=timerange) + return pairs_not_available + + def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ Get the maximum timeframe for the given backtest data diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 2d3097ec4..66a777ce5 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -28,7 +28,7 @@ class PairInfo(NamedTuple): avg_trade_duration: float -class Edge(): +class Edge: """ Calculates Win Rate, Risk Reward Ratio, Expectancy against historical data for a give set of markets and a strategy @@ -93,7 +93,7 @@ class Edge(): logger.info('Using local backtesting data (using whitelist in given config) ...') data = history.load_data( - datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, + datadir=Path(self.config['datadir']), pairs=pairs, ticker_interval=self.strategy.ticker_interval, refresh_pairs=self._refresh_pairs, diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 18e754e3f..14f409659 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -2,6 +2,10 @@ import logging from typing import Dict +import ccxt + +from freqtrade import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) @@ -25,3 +29,55 @@ class Binance(Exchange): limit = min(list(filter(lambda x: limit <= x, limit_range))) return super().get_order_book(pair, limit) + + def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: + """ + creates a stoploss limit order. + this stoploss-limit is binance-specific. + It may work with a limited number of other exchanges, but this has not been tested yet. + + """ + ordertype = "stop_loss_limit" + + stop_price = self.symbol_price_prec(pair, stop_price) + + # Ensure rate is less than stop price + if stop_price <= rate: + raise OperationalException( + 'In stoploss limit order, stop price should be more than limit price') + + if self._config['dry_run']: + dry_order = self.dry_run_order( + pair, ordertype, "sell", amount, stop_price) + return dry_order + + try: + params = self._params.copy() + params.update({'stopPrice': stop_price}) + + amount = self.symbol_amount_prec(pair, amount) + + rate = self.symbol_price_prec(pair, rate) + + order = self._api.create_order(pair, ordertype, 'sell', + amount, rate, params) + logger.info('stoploss limit order added for %s. ' + 'stop price: %s. limit: %s', pair, stop_price, rate) + return order + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to create {ordertype} sell order on market {pair}.' + f'Tried to sell amount {amount} at rate {rate}. ' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + # Errors: + # `binance Order would trigger immediately.` + raise InvalidOrderException( + f'Could not create {ordertype} sell order on market {pair}. ' + f'Tried to sell amount {amount} at rate {rate}. ' + f'Message: {e}') from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a8e974991..e2819bb59 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -14,6 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple import arrow import ccxt import ccxt.async_support as ccxt_async +from ccxt.base.decimal_to_precision import ROUND_UP, ROUND_DOWN from pandas import DataFrame from freqtrade import (DependencyException, InvalidOrderException, @@ -68,7 +69,7 @@ def retrier(f): return wrapper -class Exchange(object): +class Exchange: _config: Dict = {} _params: Dict = {} @@ -320,7 +321,7 @@ class Exchange(object): if (order_types.get("stoploss_on_exchange") and not self._ft_has.get("stoploss_on_exchange", False)): raise OperationalException( - 'On exchange stoploss is not supported for %s.' % self.name + f'On exchange stoploss is not supported for {self.name}.' ) def validate_order_time_in_force(self, order_time_in_force: Dict) -> None: @@ -366,7 +367,7 @@ class Exchange(object): def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict[str, Any]: order_id = f'dry_run_{side}_{randint(0, 10**6)}' - dry_order = { # TODO: additional entry should be added for stoploss limit + dry_order = { "id": order_id, 'pair': pair, 'price': rate, @@ -381,6 +382,7 @@ class Exchange(object): "info": {} } self._store_dry_order(dry_order) + # Copy order and close it - so the returned order is open unless it's a market order return dry_order def _store_dry_order(self, dry_order: Dict) -> None: @@ -391,6 +393,8 @@ class Exchange(object): "filled": closed_order["amount"], "remaining": 0 }) + if closed_order["type"] in ["stop_loss_limit"]: + closed_order["info"].update({"stopPrice": closed_order["price"]}) self._dry_run_open_orders[closed_order["id"]] = closed_order def create_order(self, pair: str, ordertype: str, side: str, amount: float, @@ -450,30 +454,14 @@ class Exchange(object): def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: """ creates a stoploss limit order. - NOTICE: it is not supported by all exchanges. only binance is tested for now. - TODO: implementation maybe needs to be moved to the binance subclass + Since ccxt does not unify stoploss-limit orders yet, this needs to be implemented in each + exchange's subclass. + The exception below should never raise, since we disallow + starting the bot in validate_ordertypes() + Note: Changes to this interface need to be applied to all sub-classes too. """ - ordertype = "stop_loss_limit" - stop_price = self.symbol_price_prec(pair, stop_price) - - # Ensure rate is less than stop price - if stop_price <= rate: - raise OperationalException( - 'In stoploss limit order, stop price should be more than limit price') - - if self._config['dry_run']: - dry_order = self.dry_run_order( - pair, ordertype, "sell", amount, stop_price) - return dry_order - - params = self._params.copy() - params.update({'stopPrice': stop_price}) - - order = self.create_order(pair, ordertype, 'sell', amount, rate, params) - logger.info('stoploss limit order added for %s. ' - 'stop price: %s. limit: %s', pair, stop_price, rate) - return order + raise OperationalException(f"stoploss_limit is not implemented for {self.name}.") @retrier def get_balance(self, currency: str) -> float: @@ -824,11 +812,9 @@ def timeframe_to_prev_date(timeframe: str, date: datetime = None) -> datetime: """ if not date: date = datetime.now(timezone.utc) - timeframe_secs = timeframe_to_seconds(timeframe) - # Get offset based on timerame_secs - offset = date.timestamp() % timeframe_secs - # Subtract seconds passed since last offset - new_timestamp = date.timestamp() - offset + + new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000, + ROUND_DOWN) // 1000 return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) @@ -839,9 +825,8 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: :param date: date to use. Defaults to utcnow() :returns: date of next candle (with utc timezone) """ - prevdate = timeframe_to_prev_date(timeframe, date) - timeframe_secs = timeframe_to_seconds(timeframe) - - # Add one interval to previous candle - new_timestamp = prevdate.timestamp() + timeframe_secs + if not date: + date = datetime.now(timezone.utc) + new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000, + ROUND_UP) // 1000 return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 91b41a159..6d3e82eca 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -2,7 +2,11 @@ import logging from typing import Dict +import ccxt + +from freqtrade import OperationalException, TemporaryError from freqtrade.exchange import Exchange +from freqtrade.exchange.exchange import retrier logger = logging.getLogger(__name__) @@ -10,3 +14,33 @@ logger = logging.getLogger(__name__) class Kraken(Exchange): _params: Dict = {"trading_agreement": "agree"} + + @retrier + def get_balances(self) -> dict: + if self._config['dry_run']: + return {} + + try: + balances = self._api.fetch_balance() + # Remove additional info from ccxt results + balances.pop("info", None) + balances.pop("free", None) + balances.pop("total", None) + balances.pop("used", None) + + orders = self._api.fetch_open_orders() + order_list = [(x["symbol"].split("/")[0 if x["side"] == "sell" else 1], + x["remaining"], + # Don't remove the below comment, this can be important for debuggung + # x["side"], x["amount"], + ) for x in orders] + for bal in balances: + balances[bal]['used'] = sum(order[1] for order in order_list if order[0] == bal) + balances[bal]['free'] = balances[bal]['total'] - balances[bal]['used'] + + return balances + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e88b9db6a..3f7eab27a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -6,6 +6,7 @@ import copy import logging import traceback from datetime import datetime +from math import isclose from typing import Any, Dict, List, Optional, Tuple import arrow @@ -21,7 +22,7 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.resolvers import ExchangeResolver, StrategyResolver, PairListResolver -from freqtrade.state import State, RunMode +from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.wallets import Wallets @@ -29,7 +30,7 @@ from freqtrade.wallets import Wallets logger = logging.getLogger(__name__) -class FreqtradeBot(object): +class FreqtradeBot: """ Freqtrade is the main class of the bot. This is from here the bot start its logic. @@ -79,12 +80,6 @@ class FreqtradeBot(object): persistence.init(self.config.get('db_url', None), clean_open_orders=self.config.get('dry_run', False)) - # Stoploss on exchange does not make sense, therefore we need to disable that. - if (self.dataprovider.runmode == RunMode.DRY_RUN and - self.strategy.order_types.get('stoploss_on_exchange', False)): - logger.info("Disabling stoploss_on_exchange during dry-run.") - self.strategy.order_types['stoploss_on_exchange'] = False - config['order_types']['stoploss_on_exchange'] = False # Set initial bot state from config initial_state = self.config.get('initial_state') self.state = State[initial_state.upper()] if initial_state else State.STOPPED @@ -216,7 +211,7 @@ class FreqtradeBot(object): if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.get_open_trades()) if open_trades >= self.config['max_open_trades']: - logger.warning('Can\'t open a new trade: max number of trades is reached') + logger.warning("Can't open a new trade: max number of trades is reached") return None return available_amount / (self.config['max_open_trades'] - open_trades) @@ -351,8 +346,8 @@ class FreqtradeBot(object): min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( - f'Can\'t open a new trade for {pair_s}: stake amount ' - f'is too small ({stake_amount} < {min_stake_amount})' + f"Can't open a new trade for {pair_s}: stake amount " + f"is too small ({stake_amount} < {min_stake_amount})" ) return False @@ -516,7 +511,7 @@ class FreqtradeBot(object): trade.pair.startswith(exectrade['fee']['currency'])): fee_abs += exectrade['fee']['cost'] - if amount != order_amount: + if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): logger.warning(f"Amount {amount} does not match amount {trade.amount}") raise OperationalException("Half bought? Amounts don't match") real_amount = amount - fee_abs @@ -541,7 +536,7 @@ class FreqtradeBot(object): # Try update amount (binance-fix) try: new_amount = self.get_real_amount(trade, order) - if order['amount'] != new_amount: + if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC): order['amount'] = new_amount # Fee was applied, so set to 0 trade.fee_open = 0 @@ -617,6 +612,33 @@ class FreqtradeBot(object): logger.debug('Found no sell signal for %s.', trade) return False + def create_stoploss_order(self, trade: Trade, stop_price: float, rate: float) -> bool: + """ + Abstracts creating stoploss orders from the logic. + Handles errors and updates the trade database object. + Force-sells the pair (using EmergencySell reason) in case of Problems creating the order. + :return: True if the order succeeded, and False in case of problems. + """ + # Limit price threshold: As limit price should always be below price + LIMIT_PRICE_PCT = 0.99 + + try: + stoploss_order = self.exchange.stoploss_limit(pair=trade.pair, amount=trade.amount, + stop_price=stop_price, + rate=rate * LIMIT_PRICE_PCT) + trade.stoploss_order_id = str(stoploss_order['id']) + return True + except InvalidOrderException as e: + trade.stoploss_order_id = None + logger.error(f'Unable to place a stoploss order on exchange. {e}') + logger.warning('Selling the trade forcefully') + self.execute_sell(trade, trade.stop_loss, sell_reason=SellType.EMERGENCY_SELL) + + except DependencyException: + trade.stoploss_order_id = None + logger.exception('Unable to place a stoploss order on exchange.') + return False + def handle_stoploss_on_exchange(self, trade: Trade) -> bool: """ Check if trade is fulfilled in which case the stoploss @@ -635,49 +657,25 @@ class FreqtradeBot(object): except InvalidOrderException as exception: logger.warning('Unable to fetch stoploss order: %s', exception) - # If trade open order id does not exist: buy order is fulfilled - buy_order_fulfilled = not trade.open_order_id - - # Limit price threshold: As limit price should always be below price - limit_price_pct = 0.99 - # If buy order is fulfilled but there is no stoploss, we add a stoploss on exchange - if (buy_order_fulfilled and not stoploss_order): - if self.edge: - stoploss = self.edge.stoploss(pair=trade.pair) - else: - stoploss = self.strategy.stoploss + if (not trade.open_order_id and not stoploss_order): + + stoploss = self.edge.stoploss(pair=trade.pair) if self.edge else self.strategy.stoploss stop_price = trade.open_rate * (1 + stoploss) - # limit price should be less than stop price. - limit_price = stop_price * limit_price_pct - - try: - stoploss_order_id = self.exchange.stoploss_limit( - pair=trade.pair, amount=trade.amount, stop_price=stop_price, rate=limit_price - )['id'] - trade.stoploss_order_id = str(stoploss_order_id) + if self.create_stoploss_order(trade=trade, stop_price=stop_price, rate=stop_price): trade.stoploss_last_update = datetime.now() return False - except DependencyException as exception: - trade.stoploss_order_id = None - logger.warning('Unable to place a stoploss order on exchange: %s', exception) - # If stoploss order is canceled for some reason we add it if stoploss_order and stoploss_order['status'] == 'canceled': - try: - stoploss_order_id = self.exchange.stoploss_limit( - pair=trade.pair, amount=trade.amount, - stop_price=trade.stop_loss, rate=trade.stop_loss * limit_price_pct - )['id'] - trade.stoploss_order_id = str(stoploss_order_id) + if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss, + rate=trade.stop_loss): return False - except DependencyException as exception: + else: trade.stoploss_order_id = None - logger.warning('Stoploss order was cancelled, ' - 'but unable to recreate one: %s', exception) + logger.warning('Stoploss order was cancelled, but unable to recreate one.') # We check if stoploss order is fulfilled if stoploss_order and stoploss_order['status'] == 'closed': @@ -686,7 +684,7 @@ class FreqtradeBot(object): # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['ticker_interval'])) - self._notify_sell(trade) + self._notify_sell(trade, "stoploss") return True # Finally we check if stoploss on exchange should be moved up because of trailing. @@ -720,17 +718,13 @@ class FreqtradeBot(object): logger.exception(f"Could not cancel stoploss order {order['id']} " f"for pair {trade.pair}") - try: - # creating the new one - stoploss_order_id = self.exchange.stoploss_limit( - pair=trade.pair, amount=trade.amount, - stop_price=trade.stop_loss, rate=trade.stop_loss * 0.99 - )['id'] - trade.stoploss_order_id = str(stoploss_order_id) - except DependencyException: - trade.stoploss_order_id = None - logger.exception(f"Could not create trailing stoploss order " - f"for pair {trade.pair}.") + # Create new stoploss order + if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss, + rate=trade.stop_loss): + return False + else: + logger.warning(f"Could not create trailing stoploss order " + f"for pair {trade.pair}.") def _check_and_execute_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: @@ -883,9 +877,14 @@ class FreqtradeBot(object): except InvalidOrderException: logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}") + ordertype = self.strategy.order_types[sell_type] + if sell_reason == SellType.EMERGENCY_SELL: + # Emergencysells (default to market!) + ordertype = self.strategy.order_types.get("emergencysell", "market") + # Execute sell and update trade record order = self.exchange.sell(pair=str(trade.pair), - ordertype=self.strategy.order_types[sell_type], + ordertype=ordertype, amount=trade.amount, rate=limit, time_in_force=self.strategy.order_time_in_force['sell'] ) @@ -901,9 +900,9 @@ class FreqtradeBot(object): # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['ticker_interval'])) - self._notify_sell(trade) + self._notify_sell(trade, ordertype) - def _notify_sell(self, trade: Trade): + def _notify_sell(self, trade: Trade, order_type: str): """ Sends rpc notification when a sell occured. """ @@ -920,7 +919,7 @@ class FreqtradeBot(object): 'pair': trade.pair, 'gain': gain, 'limit': trade.close_rate_requested, - 'order_type': self.strategy.order_types['sell'], + 'order_type': order_type, 'amount': trade.amount, 'open_rate': trade.open_rate, 'current_rate': current_rate, diff --git a/freqtrade/main.py b/freqtrade/main.py index a96fd43c5..4d6f0dce7 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -11,7 +11,6 @@ if sys.version_info < (3, 6): # flake8: noqa E402 import logging -from argparse import Namespace from typing import Any, List from freqtrade import OperationalException @@ -31,16 +30,13 @@ def main(sysargv: List[str] = None) -> None: return_code: Any = 1 worker = None try: - arguments = Arguments( - sysargv, - 'Free, open source crypto trading bot' - ) - args: Namespace = arguments.get_parsed_arg() + arguments = Arguments(sysargv) + args = arguments.get_parsed_arg() # A subcommand has been issued. # Means if Backtesting or Hyperopt have been called we exit the bot - if hasattr(args, 'func'): - args.func(args) + if 'func' in args: + args['func'](args) # TODO: fetch return_code as returned by the command function here return_code = 0 else: diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 12a90a14d..c9fbda17e 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -114,3 +114,10 @@ def deep_merge_dicts(source, destination): destination[key] = value return destination + + +def round_dict(d, n): + """ + Rounds float values in the dict to n digits after the decimal point. + """ + return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()} diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 2c7c42c4d..3adf5eb43 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,10 +1,7 @@ import logging -from argparse import Namespace from typing import Any, Dict -from filelock import FileLock, Timeout - -from freqtrade import DependencyException, constants +from freqtrade import DependencyException, constants, OperationalException from freqtrade.state import RunMode from freqtrade.utils import setup_utils_configuration @@ -12,7 +9,7 @@ from freqtrade.utils import setup_utils_configuration logger = logging.getLogger(__name__) -def setup_configuration(args: Namespace, method: RunMode) -> Dict[str, Any]: +def setup_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: """ Prepare the configuration for the Hyperopt module :param args: Cli args from Arguments() @@ -25,20 +22,10 @@ def setup_configuration(args: Namespace, method: RunMode) -> Dict[str, Any]: raise DependencyException('stake amount could not be "%s" for backtesting' % constants.UNLIMITED_STAKE_AMOUNT) - if method == RunMode.HYPEROPT: - # Special cases for Hyperopt - if config.get('strategy') and config.get('strategy') != 'DefaultStrategy': - logger.error("Please don't use --strategy for hyperopt.") - logger.error( - "Read the documentation at " - "https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md " - "to understand how to configure hyperopt.") - raise DependencyException("--strategy configured but not supported for hyperopt") - return config -def start_backtesting(args: Namespace) -> None: +def start_backtesting(args: Dict[str, Any]) -> None: """ Start Backtesting script :param args: Cli args from Arguments() @@ -57,15 +44,19 @@ def start_backtesting(args: Namespace) -> None: backtesting.start() -def start_hyperopt(args: Namespace) -> None: +def start_hyperopt(args: Dict[str, Any]) -> None: """ Start hyperopt script :param args: Cli args from Arguments() :return: None """ # Import here to avoid loading hyperopt module when it's not used - from freqtrade.optimize.hyperopt import Hyperopt - + try: + from filelock import FileLock, Timeout + from freqtrade.optimize.hyperopt import Hyperopt + except ImportError as e: + raise OperationalException( + f"{e}. Please ensure that the hyperopt dependencies are installed.") from e # Initialize configuration config = setup_configuration(args, RunMode.HYPEROPT) @@ -95,7 +86,7 @@ def start_hyperopt(args: Namespace) -> None: # Same in Edge and Backtesting start() functions. -def start_edge(args: Namespace) -> None: +def start_edge(args: Dict[str, Any]) -> None: """ Start Edge script :param args: Cli args from Arguments() diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 568615b53..6074b281b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -44,7 +44,7 @@ class BacktestResult(NamedTuple): sell_reason: SellType -class Backtesting(object): +class Backtesting: """ Backtesting class, this class contains all the logic to run a backtest @@ -81,6 +81,12 @@ class Backtesting(object): # No strategy list specified, only one strategy self.strategylist.append(StrategyResolver(self.config).strategy) + if "ticker_interval" not in self.config: + raise OperationalException("Ticker-interval needs to be set in either configuration " + "or as cli argument `--ticker-interval 5m`") + self.ticker_interval = str(self.config.get('ticker_interval')) + self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) + # Load one (first) strategy self._set_strategy(self.strategylist[0]) @@ -89,14 +95,6 @@ class Backtesting(object): Load strategy into backtesting """ self.strategy = strategy - if "ticker_interval" not in self.config: - raise OperationalException("Ticker-interval needs to be set in either configuration " - "or as cli argument `--ticker-interval 5m`") - - self.ticker_interval = self.config.get('ticker_interval') - self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) - self.advise_buy = strategy.advise_buy - self.advise_sell = strategy.advise_sell # Set stoploss_on_exchange to false for backtesting, # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case @@ -219,8 +217,8 @@ class Backtesting(object): for pair, pair_data in processed.items(): pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - ticker_data = self.advise_sell( - self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() + ticker_data = self.strategy.advise_sell( + self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() # to avoid using data from future, we buy/sell with signal from previous candle ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) @@ -239,14 +237,16 @@ class Backtesting(object): stake_amount: float, max_open_trades: int) -> Optional[BacktestResult]: trade = Trade( + pair=pair, open_rate=buy_row.open, open_date=buy_row.date, stake_amount=stake_amount, amount=stake_amount / buy_row.open, fee_open=self.fee, - fee_close=self.fee + fee_close=self.fee, + is_open=True, ) - + logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.") # calculate win/lose forwards from buy point for sell_row in partial_ticker: if max_open_trades > 0: @@ -289,23 +289,25 @@ class Backtesting(object): if partial_ticker: # no sell condition found - trade stil open at end of backtest period sell_row = partial_ticker[-1] - btr = BacktestResult(pair=pair, - profit_percent=trade.calc_profit_percent(rate=sell_row.open), - profit_abs=trade.calc_profit(rate=sell_row.open), - open_time=buy_row.date, - close_time=sell_row.date, - trade_duration=int(( - sell_row.date - buy_row.date).total_seconds() // 60), - open_index=buy_row.Index, - close_index=sell_row.Index, - open_at_end=True, - open_rate=buy_row.open, - close_rate=sell_row.open, - sell_reason=SellType.FORCE_SELL - ) - logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, - btr.profit_percent, btr.profit_abs) - return btr + bt_res = BacktestResult(pair=pair, + profit_percent=trade.calc_profit_percent(rate=sell_row.open), + profit_abs=trade.calc_profit(rate=sell_row.open), + open_time=buy_row.date, + close_time=sell_row.date, + trade_duration=int(( + sell_row.date - buy_row.date).total_seconds() // 60), + open_index=buy_row.Index, + close_index=sell_row.Index, + open_at_end=True, + open_rate=buy_row.open, + close_rate=sell_row.open, + sell_reason=SellType.FORCE_SELL + ) + logger.debug(f"{pair} - Force selling still open trade, " + f"profit percent: {bt_res.profit_percent}, " + f"profit abs: {bt_res.profit_abs}") + + return bt_res return None def backtest(self, args: Dict) -> DataFrame: @@ -384,6 +386,8 @@ class Backtesting(object): max_open_trades) if trade_entry: + logger.debug(f"{pair} - Locking pair till " + f"close_time={trade_entry.close_time}") lock_pair_until[pair] = trade_entry.close_time trades.append(trade_entry) else: @@ -407,11 +411,9 @@ class Backtesting(object): timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = history.load_data( - datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, + datadir=Path(self.config['datadir']), pairs=pairs, ticker_interval=self.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.exchange, timerange=timerange, ) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 7e0d60843..0cf5a009b 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -16,7 +16,7 @@ from freqtrade.resolvers import StrategyResolver logger = logging.getLogger(__name__) -class EdgeCli(object): +class EdgeCli: """ EdgeCli class, this class contains all the logic to run edge backtesting @@ -39,7 +39,8 @@ class EdgeCli(object): self.strategy = StrategyResolver(self.config).strategy self.edge = Edge(config, self.exchange, self.strategy) - self.edge._refresh_pairs = self.config.get('refresh_pairs', False) + # Set refresh_pairs to false for edge-cli (it must be true for edge) + self.edge._refresh_pairs = False self.timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 62a6ab27b..a70ff8142 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -24,8 +24,10 @@ from skopt.space import Dimension from freqtrade.configuration import TimeRange from freqtrade.data.history import load_data, get_timeframe +from freqtrade.misc import round_dict from freqtrade.optimize.backtesting import Backtesting -# Import IHyperOptLoss to allow users import from this file +# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules +from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F4 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F4 from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver, HyperOptLossResolver @@ -34,6 +36,11 @@ logger = logging.getLogger(__name__) INITIAL_POINTS = 30 + +# Keep no more than 2*SKOPT_MODELS_MAX_NUM models +# in the skopt models list +SKOPT_MODELS_MAX_NUM = 10 + MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization @@ -47,10 +54,11 @@ class Hyperopt: """ def __init__(self, config: Dict[str, Any]) -> None: self.config = config - self.backtesting = Backtesting(self.config) self.custom_hyperopt = HyperOptResolver(self.config).hyperopt + self.backtesting = Backtesting(self.config) + self.custom_hyperoptloss = HyperOptLossResolver(self.config).hyperoptloss self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function @@ -71,11 +79,15 @@ class Hyperopt: self.trials: List = [] # Populate functions here (hasattr is slow so should not be run during "regular" operations) + if hasattr(self.custom_hyperopt, 'populate_indicators'): + self.backtesting.strategy.advise_indicators = \ + self.custom_hyperopt.populate_indicators # type: ignore if hasattr(self.custom_hyperopt, 'populate_buy_trend'): - self.backtesting.advise_buy = self.custom_hyperopt.populate_buy_trend # type: ignore - + self.backtesting.strategy.advise_buy = \ + self.custom_hyperopt.populate_buy_trend # type: ignore if hasattr(self.custom_hyperopt, 'populate_sell_trend'): - self.backtesting.advise_sell = self.custom_hyperopt.populate_sell_trend # type: ignore + self.backtesting.strategy.advise_sell = \ + self.custom_hyperopt.populate_sell_trend # type: ignore # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): @@ -83,7 +95,7 @@ class Hyperopt: else: logger.debug('Ignoring max_open_trades (--disable-max-market-positions was used) ...') self.max_open_trades = 0 - self.position_stacking = self.config.get('position_stacking', False), + self.position_stacking = self.config.get('position_stacking', False) if self.has_space('sell'): # Make sure experimental is enabled @@ -107,7 +119,9 @@ class Hyperopt: p.unlink() def get_args(self, params): - dimensions = self.hyperopt_space() + + dimensions = self.dimensions + # Ensure the number of dimensions match # the number of parameters in the list x. if len(params) != len(dimensions): @@ -124,14 +138,14 @@ class Hyperopt: Save hyperopt trials to file """ if self.trials: - logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file) + logger.info("Saving %d evaluations to '%s'", len(self.trials), self.trials_file) dump(self.trials, self.trials_file) def read_trials(self) -> List: """ Read hyperopt trials file """ - logger.info('Reading Trials from \'%s\'', self.trials_file) + logger.info("Reading Trials from '%s'", self.trials_file) trials = load(self.trials_file) self.trials_file.unlink() return trials @@ -178,9 +192,11 @@ class Hyperopt: indent=4) if self.has_space('roi'): print("ROI table:") - pprint(self.custom_hyperopt.generate_roi_table(params), indent=4) + # Round printed values to 5 digits after the decimal point + pprint(round_dict(self.custom_hyperopt.generate_roi_table(params), 5), indent=4) if self.has_space('stoploss'): - print(f"Stoploss: {params.get('stoploss')}") + # Also round to 5 digits after the decimal point + print(f"Stoploss: {round(params.get('stoploss'), 5)}") def log_results(self, results) -> None: """ @@ -244,20 +260,24 @@ class Hyperopt: spaces += self.custom_hyperopt.stoploss_space() return spaces - def generate_optimizer(self, _params: Dict) -> Dict: + def generate_optimizer(self, _params: Dict, iteration=None) -> Dict: """ Used Optimize function. Called once per epoch to optimize whatever is configured. Keep this function as optimized as possible! """ params = self.get_args(_params) + if self.has_space('roi'): - self.backtesting.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) + self.backtesting.strategy.minimal_roi = \ + self.custom_hyperopt.generate_roi_table(params) if self.has_space('buy'): - self.backtesting.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) + self.backtesting.strategy.advise_buy = \ + self.custom_hyperopt.buy_strategy_generator(params) if self.has_space('sell'): - self.backtesting.advise_sell = self.custom_hyperopt.sell_strategy_generator(params) + self.backtesting.strategy.advise_sell = \ + self.custom_hyperopt.sell_strategy_generator(params) if self.has_space('stoploss'): self.backtesting.strategy.stoploss = params['stoploss'] @@ -318,9 +338,9 @@ class Hyperopt: f'Total profit {total_profit: 11.8f} {stake_cur} ' f'({profit: 7.2f}Σ%). Avg duration {duration:5.1f} mins.') - def get_optimizer(self, cpu_count) -> Optimizer: + def get_optimizer(self, dimensions, cpu_count) -> Optimizer: return Optimizer( - self.hyperopt_space(), + dimensions, base_estimator="ET", acq_optimizer="auto", n_initial_points=INITIAL_POINTS, @@ -328,9 +348,26 @@ class Hyperopt: random_state=self.config.get('hyperopt_random_state', None) ) - def run_optimizer_parallel(self, parallel, asked) -> List: + def fix_optimizer_models_list(self): + """ + WORKAROUND: Since skopt is not actively supported, this resolves problems with skopt + memory usage, see also: https://github.com/scikit-optimize/scikit-optimize/pull/746 + + This may cease working when skopt updates if implementation of this intrinsic + part changes. + """ + n = len(self.opt.models) - SKOPT_MODELS_MAX_NUM + # Keep no more than 2*SKOPT_MODELS_MAX_NUM models in the skopt models list, + # remove the old ones. These are actually of no use, the current model + # from the estimator is the only one used in the skopt optimizer. + # Freqtrade code also does not inspect details of the models. + if n >= SKOPT_MODELS_MAX_NUM: + logger.debug(f"Fixing skopt models list, removing {n} old items...") + del self.opt.models[0:n] + + def run_optimizer_parallel(self, parallel, asked, i) -> List: return parallel(delayed( - wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) + wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) def load_previous_results(self): """ read trials file if we have one """ @@ -345,11 +382,9 @@ class Hyperopt: timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = load_data( - datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, + datadir=Path(self.config['datadir']), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.backtesting.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.backtesting.exchange, timerange=timerange ) @@ -366,9 +401,6 @@ class Hyperopt: (max_date - min_date).days ) - self.backtesting.strategy.advise_indicators = \ - self.custom_hyperopt.populate_indicators # type: ignore - preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data) dump(preprocessed, self.tickerdata_pickle) @@ -379,11 +411,12 @@ class Hyperopt: self.load_previous_results() cpus = cpu_count() - logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') + logger.info(f"Found {cpus} CPU cores. Let's make them scream!") config_jobs = self.config.get('hyperopt_jobs', -1) logger.info(f'Number of parallel jobs set as: {config_jobs}') - opt = self.get_optimizer(config_jobs) + self.dimensions = self.hyperopt_space() + self.opt = self.get_optimizer(self.dimensions, config_jobs) if self.config.get('print_colorized', False): colorama_init(autoreset=True) @@ -394,9 +427,10 @@ class Hyperopt: logger.info(f'Effective number of parallel workers used: {jobs}') EVALS = max(self.total_epochs // jobs, 1) for i in range(EVALS): - asked = opt.ask(n_points=jobs) - f_val = self.run_optimizer_parallel(parallel, asked) - opt.tell(asked, [v['loss'] for v in f_val]) + asked = self.opt.ask(n_points=jobs) + f_val = self.run_optimizer_parallel(parallel, asked, i) + self.opt.tell(asked, [v['loss'] for v in f_val]) + self.fix_optimizer_models_list() for j in range(jobs): current = i * jobs + j val = f_val[j] diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index f1f123653..4208b29d3 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -2,6 +2,8 @@ IHyperOpt interface This module defines the interface to apply for hyperopts """ +import logging +import math from abc import ABC, abstractmethod from typing import Dict, Any, Callable, List @@ -9,19 +11,37 @@ from typing import Dict, Any, Callable, List from pandas import DataFrame from skopt.space import Dimension, Integer, Real +from freqtrade import OperationalException +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.misc import round_dict + + +logger = logging.getLogger(__name__) + + +def _format_exception_message(method: str, space: str) -> str: + return (f"The '{space}' space is included into the hyperoptimization " + f"but {method}() method is not found in your " + f"custom Hyperopt class. You should either implement this " + f"method or remove the '{space}' space from hyperoptimization.") + class IHyperOpt(ABC): """ Interface for freqtrade hyperopts - Defines the mandatory structure must follow any custom strategies + Defines the mandatory structure must follow any custom hyperopts - Attributes you can use: - minimal_roi -> Dict: Minimal ROI designed for the strategy - stoploss -> float: optimal stoploss designed for the strategy + Class attributes you can use: ticker_interval -> int: value of the ticker interval to use for the strategy """ ticker_interval: str + def __init__(self, config: dict) -> None: + self.config = config + + # Assign ticker_interval to be used in hyperopt + IHyperOpt.ticker_interval = str(config['ticker_interval']) + @staticmethod @abstractmethod def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -32,32 +52,32 @@ class IHyperOpt(ABC): """ @staticmethod - @abstractmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Create a buy strategy generator. """ + raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy')) @staticmethod - @abstractmethod def sell_strategy_generator(params: Dict[str, Any]) -> Callable: """ Create a sell strategy generator. """ + raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell')) @staticmethod - @abstractmethod def indicator_space() -> List[Dimension]: """ Create an indicator space. """ + raise OperationalException(_format_exception_message('indicator_space', 'buy')) @staticmethod - @abstractmethod def sell_indicator_space() -> List[Dimension]: """ Create a sell indicator space. """ + raise OperationalException(_format_exception_message('sell_indicator_space', 'sell')) @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: @@ -75,6 +95,83 @@ class IHyperOpt(ABC): return roi_table + @staticmethod + def roi_space() -> List[Dimension]: + """ + Create a ROI space. + + Defines values to search for each ROI steps. + + This method implements adaptive roi hyperspace with varied + ranges for parameters which automatically adapts to the + ticker interval used. + + It's used by Freqtrade by default, if no custom roi_space method is defined. + """ + + # Default scaling coefficients for the roi hyperspace. Can be changed + # to adjust resulting ranges of the ROI tables. + # Increase if you need wider ranges in the roi hyperspace, decrease if shorter + # ranges are needed. + roi_t_alpha = 1.0 + roi_p_alpha = 1.0 + + ticker_interval_mins = timeframe_to_minutes(IHyperOpt.ticker_interval) + + # We define here limits for the ROI space parameters automagically adapted to the + # ticker_interval used by the bot: + # + # * 'roi_t' (limits for the time intervals in the ROI tables) components + # are scaled linearly. + # * 'roi_p' (limits for the ROI value steps) components are scaled logarithmically. + # + # The scaling is designed so that it maps exactly to the legacy Freqtrade roi_space() + # method for the 5m ticker interval. + roi_t_scale = ticker_interval_mins / 5 + roi_p_scale = math.log1p(ticker_interval_mins) / math.log1p(5) + roi_limits = { + 'roi_t1_min': int(10 * roi_t_scale * roi_t_alpha), + 'roi_t1_max': int(120 * roi_t_scale * roi_t_alpha), + 'roi_t2_min': int(10 * roi_t_scale * roi_t_alpha), + 'roi_t2_max': int(60 * roi_t_scale * roi_t_alpha), + 'roi_t3_min': int(10 * roi_t_scale * roi_t_alpha), + 'roi_t3_max': int(40 * roi_t_scale * roi_t_alpha), + 'roi_p1_min': 0.01 * roi_p_scale * roi_p_alpha, + 'roi_p1_max': 0.04 * roi_p_scale * roi_p_alpha, + 'roi_p2_min': 0.01 * roi_p_scale * roi_p_alpha, + 'roi_p2_max': 0.07 * roi_p_scale * roi_p_alpha, + 'roi_p3_min': 0.01 * roi_p_scale * roi_p_alpha, + 'roi_p3_max': 0.20 * roi_p_scale * roi_p_alpha, + } + logger.debug(f"Using roi space limits: {roi_limits}") + p = { + 'roi_t1': roi_limits['roi_t1_min'], + 'roi_t2': roi_limits['roi_t2_min'], + 'roi_t3': roi_limits['roi_t3_min'], + 'roi_p1': roi_limits['roi_p1_min'], + 'roi_p2': roi_limits['roi_p2_min'], + 'roi_p3': roi_limits['roi_p3_min'], + } + logger.info(f"Min roi table: {round_dict(IHyperOpt.generate_roi_table(p), 5)}") + p = { + 'roi_t1': roi_limits['roi_t1_max'], + 'roi_t2': roi_limits['roi_t2_max'], + 'roi_t3': roi_limits['roi_t3_max'], + 'roi_p1': roi_limits['roi_p1_max'], + 'roi_p2': roi_limits['roi_p2_max'], + 'roi_p3': roi_limits['roi_p3_max'], + } + logger.info(f"Max roi table: {round_dict(IHyperOpt.generate_roi_table(p), 5)}") + + return [ + Integer(roi_limits['roi_t1_min'], roi_limits['roi_t1_max'], name='roi_t1'), + Integer(roi_limits['roi_t2_min'], roi_limits['roi_t2_max'], name='roi_t2'), + Integer(roi_limits['roi_t3_min'], roi_limits['roi_t3_max'], name='roi_t3'), + Real(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], name='roi_p1'), + Real(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], name='roi_p2'), + Real(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], name='roi_p3'), + ] + @staticmethod def stoploss_space() -> List[Dimension]: """ @@ -84,22 +181,17 @@ class IHyperOpt(ABC): You may override it in your custom Hyperopt class. """ return [ - Real(-0.5, -0.02, name='stoploss'), + Real(-0.35, -0.02, name='stoploss'), ] - @staticmethod - def roi_space() -> List[Dimension]: - """ - Create a ROI space. + # This is needed for proper unpickling the class attribute ticker_interval + # which is set to the actual value by the resolver. + # Why do I still need such shamanic mantras in modern python? + def __getstate__(self): + state = self.__dict__.copy() + state['ticker_interval'] = self.ticker_interval + return state - Defines values to search for each ROI steps. - You may override it in your custom Hyperopt class. - """ - return [ - Integer(10, 120, name='roi_t1'), - Integer(10, 60, name='roi_t2'), - Integer(10, 40, name='roi_t3'), - Real(0.01, 0.04, name='roi_p1'), - Real(0.01, 0.07, name='roi_p2'), - Real(0.01, 0.20, name='roi_p3'), - ] + def __setstate__(self, state): + self.__dict__.update(state) + IHyperOpt.ticker_interval = state['ticker_interval'] diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c844bbc4c..1850aafd9 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -1,7 +1,6 @@ """ This module contains the class to persist trades into SQLite """ - import logging from datetime import datetime from decimal import Decimal @@ -19,8 +18,10 @@ from sqlalchemy.pool import StaticPool from freqtrade import OperationalException + logger = logging.getLogger(__name__) + _DECL_BASE: Any = declarative_base() _SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' @@ -48,8 +49,8 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: try: engine = create_engine(db_url, **kwargs) except NoSuchModuleError: - raise OperationalException(f'Given value for db_url: \'{db_url}\' ' - f'is no valid database URL! (See {_SQL_DOCS_URL})') + raise OperationalException(f"Given value for db_url: '{db_url}' " + f"is no valid database URL! (See {_SQL_DOCS_URL})") session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() @@ -209,7 +210,7 @@ class Trade(_DECL_BASE): ticker_interval = Column(Integer, nullable=True) def __repr__(self): - open_since = arrow.get(self.open_date).humanize() if self.is_open else 'closed' + open_since = self.open_date.strftime('%Y-%m-%d %H:%M:%S') if self.is_open else 'closed' return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' f'open_rate={self.open_rate:.8f}, open_since={open_since})') @@ -250,7 +251,6 @@ class Trade(_DECL_BASE): :param initial: Called to initiate stop_loss. Skips everything if self.stop_loss is already set. """ - if initial and not (self.stop_loss is None or self.stop_loss == 0): # Don't modify if called with initial and nothing to do return @@ -259,7 +259,7 @@ class Trade(_DECL_BASE): # no stop loss assigned yet if not self.stop_loss: - logger.debug("assigning new stop loss") + logger.debug(f"{self.pair} - Assigning new stoploss...") self.stop_loss = new_loss self.stop_loss_pct = -1 * abs(stoploss) self.initial_stop_loss = new_loss @@ -269,21 +269,20 @@ class Trade(_DECL_BASE): # evaluate if the stop loss needs to be updated else: if new_loss > self.stop_loss: # stop losses only walk up, never down! + logger.debug(f"{self.pair} - Adjusting stoploss...") self.stop_loss = new_loss self.stop_loss_pct = -1 * abs(stoploss) self.stoploss_last_update = datetime.utcnow() - logger.debug("adjusted stop loss") else: - logger.debug("keeping current stop loss") + logger.debug(f"{self.pair} - Keeping current stoploss...") logger.debug( - f"{self.pair} - current price {current_price:.8f}, " - f"bought at {self.open_rate:.8f} and calculated " - f"stop loss is at: {self.initial_stop_loss:.8f} initial " - f"stop at {self.stop_loss:.8f}. " - f"trailing stop loss saved us: " - f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f} " - f"and max observed rate was {self.max_rate:.8f}") + f"{self.pair} - Stoploss adjusted. current_price={current_price:.8f}, " + f"open_rate={self.open_rate:.8f}, max_rate={self.max_rate:.8f}, " + f"initial_stop_loss={self.initial_stop_loss:.8f}, " + f"stop_loss={self.stop_loss:.8f}. " + f"Trailing stoploss saved us: " + f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f}.") def update(self, order: Dict) -> None: """ @@ -331,24 +330,19 @@ class Trade(_DECL_BASE): self ) - def calc_open_trade_price( - self, - fee: Optional[float] = None) -> float: + def calc_open_trade_price(self, fee: Optional[float] = None) -> float: """ Calculate the open_rate including fee. :param fee: fee to use on the open rate (optional). If rate is not set self.fee will be used :return: Price in of the open trade incl. Fees """ - buy_trade = (Decimal(self.amount) * Decimal(self.open_rate)) fees = buy_trade * Decimal(fee or self.fee_open) return float(buy_trade + fees) - def calc_close_trade_price( - self, - rate: Optional[float] = None, - fee: Optional[float] = None) -> float: + def calc_close_trade_price(self, rate: Optional[float] = None, + fee: Optional[float] = None) -> float: """ Calculate the close_rate including fee :param fee: fee to use on the close rate (optional). @@ -357,7 +351,6 @@ class Trade(_DECL_BASE): If rate is not set self.close_rate will be used :return: Price in BTC of the open trade """ - if rate is None and not self.close_rate: return 0.0 @@ -365,10 +358,8 @@ class Trade(_DECL_BASE): fees = sell_trade * Decimal(fee or self.fee_close) return float(sell_trade - fees) - def calc_profit( - self, - rate: Optional[float] = None, - fee: Optional[float] = None) -> float: + def calc_profit(self, rate: Optional[float] = None, + fee: Optional[float] = None) -> float: """ Calculate the absolute profit in stake currency between Close and Open trade :param fee: fee to use on the close rate (optional). @@ -385,10 +376,8 @@ class Trade(_DECL_BASE): profit = close_trade_price - open_trade_price return float(f"{profit:.8f}") - def calc_profit_percent( - self, - rate: Optional[float] = None, - fee: Optional[float] = None) -> float: + def calc_profit_percent(self, rate: Optional[float] = None, + fee: Optional[float] = None) -> float: """ Calculates the profit in percentage (including fee). :param rate: rate to compare with (optional). @@ -396,7 +385,6 @@ class Trade(_DECL_BASE): :param fee: fee to use on the close rate (optional). :return: profit in percentage as float """ - open_trade_price = self.calc_open_trade_price() close_trade_price = self.calc_close_trade_price( rate=(rate or self.close_rate), @@ -436,8 +424,8 @@ class Trade(_DECL_BASE): and trade.initial_stop_loss_pct != desired_stoploss): # Stoploss value got changed - logger.info(f"Stoploss for {trade} needs adjustment.") + logger.info(f"Stoploss for {trade} needs adjustment...") # Force reset of stoploss trade.stop_loss = None trade.adjust_stop_loss(trade.open_rate, desired_stoploss) - logger.info(f"new stoploss: {trade.stop_loss}, ") + logger.info(f"New stoploss: {trade.stop_loss}.") diff --git a/freqtrade/plot/plot_utils.py b/freqtrade/plot/plot_utils.py new file mode 100644 index 000000000..8de0eb9e7 --- /dev/null +++ b/freqtrade/plot/plot_utils.py @@ -0,0 +1,36 @@ +from typing import Any, Dict + +from freqtrade import OperationalException +from freqtrade.state import RunMode +from freqtrade.utils import setup_utils_configuration + + +def validate_plot_args(args: Dict[str, Any]): + if not args.get('datadir') and not args.get('config'): + raise OperationalException( + "You need to specify either `--datadir` or `--config` " + "for plot-profit and plot-dataframe.") + + +def start_plot_dataframe(args: Dict[str, Any]) -> None: + """ + Entrypoint for dataframe plotting + """ + # Import here to avoid errors if plot-dependencies are not installed. + from freqtrade.plot.plotting import load_and_plot_trades + validate_plot_args(args) + config = setup_utils_configuration(args, RunMode.PLOT) + + load_and_plot_trades(config) + + +def start_plot_profit(args: Dict[str, Any]) -> None: + """ + Entrypoint for plot_profit + """ + # Import here to avoid errors if plot-dependencies are not installed. + from freqtrade.plot.plotting import plot_profit + validate_plot_args(args) + config = setup_utils_configuration(args, RunMode.PLOT) + + plot_profit(config) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 9dc6b9551..1627959f9 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -1,15 +1,14 @@ import logging from pathlib import Path -from typing import Dict, List, Optional +from typing import Any, Dict, List import pandas as pd - from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import (combine_tickers_with_mean, - create_cum_profit, load_trades) -from freqtrade.exchange import Exchange -from freqtrade.resolvers import ExchangeResolver, StrategyResolver + create_cum_profit, + extract_trades_of_period, load_trades) +from freqtrade.resolvers import StrategyResolver logger = logging.getLogger(__name__) @@ -19,23 +18,16 @@ try: from plotly.offline import plot import plotly.graph_objects as go except ImportError: - logger.exception("Module plotly not found \n Please install using `pip install plotly`") + logger.exception("Module plotly not found \n Please install using `pip3 install plotly`") exit(1) def init_plotscript(config): """ Initialize objects needed for plotting - :return: Dict with tickers, trades, pairs and strategy + :return: Dict with tickers, trades and pairs """ - exchange: Optional[Exchange] = None - # Exchange is only needed when downloading data! - if config.get("refresh_pairs", False): - exchange = ExchangeResolver(config.get('exchange', {}).get('name'), - config).exchange - - strategy = StrategyResolver(config).strategy if "pairs" in config: pairs = config["pairs"] else: @@ -47,17 +39,18 @@ def init_plotscript(config): tickers = history.load_data( datadir=Path(str(config.get("datadir"))), pairs=pairs, - ticker_interval=config['ticker_interval'], - refresh_pairs=config.get('refresh_pairs', False), + ticker_interval=config.get('ticker_interval', '5m'), timerange=timerange, - exchange=exchange, ) - trades = load_trades(config) + trades = load_trades(config['trade_source'], + db_url=config.get('db_url'), + exportfilename=config.get('exportfilename'), + ) + return {"tickers": tickers, "trades": trades, "pairs": pairs, - "strategy": strategy, } @@ -280,8 +273,15 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], name='Avg close price', ) - fig = make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1]) - fig['layout'].update(title="Profit plot") + fig = make_subplots(rows=3, cols=1, shared_xaxes=True, + row_width=[1, 1, 1], + vertical_spacing=0.05, + subplot_titles=["AVG Close Price", "Combined Profit", "Profit per pair"]) + fig['layout'].update(title="Freqtrade Profit plot") + fig['layout']['yaxis1'].update(title='Price') + fig['layout']['yaxis2'].update(title='Profit') + fig['layout']['yaxis3'].update(title='Profit') + fig['layout']['xaxis']['rangeslider'].update(visible=False) fig.add_trace(avgclose, 1, 1) fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit') @@ -321,3 +321,65 @@ def store_plot_file(fig, filename: str, directory: Path, auto_open: bool = False plot(fig, filename=str(_filename), auto_open=auto_open) logger.info(f"Stored plot as {_filename}") + + +def load_and_plot_trades(config: Dict[str, Any]): + """ + From configuration provided + - Initializes plot-script + - Get tickers data + - Generate Dafaframes populated with indicators and signals based on configured strategy + - Load trades excecuted during the selected period + - Generate Plotly plot objects + - Generate plot files + :return: None + """ + strategy = StrategyResolver(config).strategy + + plot_elements = init_plotscript(config) + trades = plot_elements['trades'] + pair_counter = 0 + for pair, data in plot_elements["tickers"].items(): + pair_counter += 1 + logger.info("analyse pair %s", pair) + tickers = {} + tickers[pair] = data + + dataframe = strategy.analyze_ticker(tickers[pair], {'pair': pair}) + trades_pair = trades.loc[trades['pair'] == pair] + trades_pair = extract_trades_of_period(dataframe, trades_pair) + + fig = generate_candlestick_graph( + pair=pair, + data=dataframe, + trades=trades_pair, + indicators1=config["indicators1"], + indicators2=config["indicators2"], + ) + + store_plot_file(fig, filename=generate_plot_filename(pair, config['ticker_interval']), + directory=config['user_data_dir'] / "plot") + + logger.info('End of plotting process. %s plots generated', pair_counter) + + +def plot_profit(config: Dict[str, Any]) -> None: + """ + Plots the total profit for all pairs. + Note, the profit calculation isn't realistic. + But should be somewhat proportional, and therefor useful + in helping out to find a good algorithm. + """ + plot_elements = init_plotscript(config) + trades = load_trades(config['trade_source'], + db_url=str(config.get('db_url')), + exportfilename=str(config.get('exportfilename')), + ) + # Filter trades to relevant pairs + trades = trades[trades['pair'].isin(plot_elements["pairs"])] + + # Create an average close price of all the pairs that were involved. + # this could be useful to gauge the overall market trend + fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades) + store_plot_file(fig, filename='freqtrade-profit-plot.html', + directory=config['user_data_dir'] / "plot", auto_open=True) diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 15d1997ef..e96394d69 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -34,15 +34,12 @@ class HyperOptResolver(IResolver): self.hyperopt = self._load_hyperopt(hyperopt_name, config, extra_dir=config.get('hyperopt_path')) - # Assign ticker_interval to be used in hyperopt - self.hyperopt.__class__.ticker_interval = str(config['ticker_interval']) - if not hasattr(self.hyperopt, 'populate_buy_trend'): - logger.warning("Custom Hyperopt does not provide populate_buy_trend. " - "Using populate_buy_trend from DefaultStrategy.") + logger.warning("Hyperopt class does not provide populate_buy_trend() method. " + "Using populate_buy_trend from the strategy.") if not hasattr(self.hyperopt, 'populate_sell_trend'): - logger.warning("Custom Hyperopt does not provide populate_sell_trend. " - "Using populate_sell_trend from DefaultStrategy.") + logger.warning("Hyperopt class does not provide populate_sell_trend() method. " + "Using populate_sell_trend from the strategy.") def _load_hyperopt( self, hyperopt_name: str, config: Dict, extra_dir: Optional[str] = None) -> IHyperOpt: @@ -65,7 +62,7 @@ class HyperOptResolver(IResolver): abs_paths.insert(0, Path(extra_dir).resolve()) hyperopt = self._load_object(paths=abs_paths, object_type=IHyperOpt, - object_name=hyperopt_name) + object_name=hyperopt_name, kwargs={'config': config}) if hyperopt: return hyperopt raise OperationalException( diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 8d93af568..6303d4801 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -12,7 +12,7 @@ from typing import Any, List, Optional, Tuple, Type, Union logger = logging.getLogger(__name__) -class IResolver(object): +class IResolver: """ This class contains all the logic to load custom classes """ diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 37aa96b68..ca7e1165b 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -13,7 +13,6 @@ from typing import Dict, Optional from freqtrade import constants, OperationalException from freqtrade.resolvers import IResolver -from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) @@ -153,13 +152,12 @@ class StrategyResolver(IResolver): strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args) + if any([x == 2 for x in [strategy._populate_fun_len, + strategy._buy_fun_len, + strategy._sell_fun_len]]): + strategy.INTERFACE_VERSION = 1 - try: - return import_strategy(strategy, config=config) - except TypeError as e: - logger.warning( - f"Impossible to load strategy '{strategy_name}'. " - f"Error: {e}") + return strategy raise OperationalException( f"Impossible to load Strategy '{strategy_name}'. This class does not exist " diff --git a/freqtrade/rpc/fiat_convert.py b/freqtrade/rpc/fiat_convert.py index 6812bf77f..d40f9221e 100644 --- a/freqtrade/rpc/fiat_convert.py +++ b/freqtrade/rpc/fiat_convert.py @@ -15,7 +15,7 @@ from freqtrade.constants import SUPPORTED_FIAT logger = logging.getLogger(__name__) -class CryptoFiat(object): +class CryptoFiat: """ Object to describe what is the price of Crypto-currency in a FIAT """ @@ -60,7 +60,7 @@ class CryptoFiat(object): return self._expiration - time.time() <= 0 -class CryptoToFiatConverter(object): +class CryptoToFiatConverter: """ Main class to initiate Crypto to FIAT. This object contains a list of pair Crypto, FIAT @@ -104,7 +104,7 @@ class CryptoToFiatConverter(object): :return: float, value in fiat of the crypto-currency amount """ if crypto_symbol == fiat_symbol: - return crypto_amount + return float(crypto_amount) price = self.get_price(crypto_symbol=crypto_symbol, fiat_symbol=fiat_symbol) return float(crypto_amount) * float(price) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 7b811cadc..f994ac006 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -54,7 +54,7 @@ class RPCException(Exception): } -class RPC(object): +class RPC: """ RPC class can be used to have extra feature, like bot data, and access to DB data """ @@ -294,9 +294,9 @@ class RPC(object): total = total + est_btc output.append({ 'currency': coin, - 'available': balance['free'], - 'balance': balance['total'], - 'pending': balance['used'], + 'free': balance['free'] if balance['free'] is not None else 0, + 'balance': balance['total'] if balance['total'] is not None else 0, + 'used': balance['used'] if balance['used'] is not None else 0, 'est_btc': est_btc, }) if total == 0.0: diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index fad532aa0..802550b94 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -9,7 +9,7 @@ from freqtrade.rpc import RPC, RPCMessageType logger = logging.getLogger(__name__) -class RPCManager(object): +class RPCManager: """ Class to manage RPC objects (Telegram, Slack, ...) """ @@ -56,7 +56,10 @@ class RPCManager(object): logger.info('Sending rpc message: %s', msg) for mod in self.registered_modules: logger.debug('Forwarding message to rpc.%s', mod.name) - mod.send_msg(msg) + try: + mod.send_msg(msg) + except NotImplementedError: + logger.error(f"Message type {msg['type']} not implemented by handler {mod.name}.") def startup_messages(self, config, pairlist) -> None: if config.get('dry_run', False): diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index fe4929780..80582a0ce 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -4,12 +4,12 @@ This module manage Telegram communication """ import logging -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict from tabulate import tabulate -from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update +from telegram import ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError -from telegram.ext import CommandHandler, Updater +from telegram.ext import CommandHandler, Updater, CallbackContext from freqtrade.__init__ import __version__ from freqtrade.rpc import RPC, RPCException, RPCMessageType @@ -31,7 +31,7 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: """ def wrapper(self, *args, **kwargs): """ Decorator logic """ - update = kwargs.get('update') or args[1] + update = kwargs.get('update') or args[0] # Reject unauthorized messages chat_id = int(self._config['telegram']['chat_id']) @@ -79,7 +79,8 @@ class Telegram(RPC): registers all known command handlers and starts polling for message updates """ - self._updater = Updater(token=self._config['telegram']['token'], workers=0) + self._updater = Updater(token=self._config['telegram']['token'], workers=0, + use_context=True) # Register command handler and start telegram message polling handles = [ @@ -96,7 +97,7 @@ class Telegram(RPC): CommandHandler('reload_conf', self._reload_conf), CommandHandler('stopbuy', self._stopbuy), CommandHandler('whitelist', self._whitelist), - CommandHandler('blacklist', self._blacklist, pass_args=True), + CommandHandler('blacklist', self._blacklist), CommandHandler('edge', self._edge), CommandHandler('help', self._help), CommandHandler('version', self._version), @@ -175,7 +176,7 @@ class Telegram(RPC): self._send_msg(message) @authorized_only - def _status(self, bot: Bot, update: Update) -> None: + def _status(self, update: Update, context: CallbackContext) -> None: """ Handler for /status. Returns the current TradeThread status @@ -184,11 +185,8 @@ class Telegram(RPC): :return: None """ - # Check if additional parameters are passed - params = update.message.text.replace('/status', '').split(' ') \ - if update.message.text else [] - if 'table' in params: - self._status_table(bot, update) + if 'table' in context.args: + self._status_table(update, context) return try: @@ -221,13 +219,13 @@ class Telegram(RPC): messages.append("\n".join([l for l in lines if l]).format(**r)) for msg in messages: - self._send_msg(msg, bot=bot) + self._send_msg(msg) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _status_table(self, bot: Bot, update: Update) -> None: + def _status_table(self, update: Update, context: CallbackContext) -> None: """ Handler for /status table. Returns the current TradeThread status in table format @@ -240,10 +238,10 @@ class Telegram(RPC): message = tabulate(df_statuses, headers='keys', tablefmt='simple') self._send_msg(f"
{message}
", parse_mode=ParseMode.HTML) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _daily(self, bot: Bot, update: Update) -> None: + def _daily(self, update: Update, context: CallbackContext) -> None: """ Handler for /daily Returns a daily profit (in BTC) over the last n days. @@ -254,8 +252,8 @@ class Telegram(RPC): stake_cur = self._config['stake_currency'] fiat_disp_cur = self._config.get('fiat_display_currency', '') try: - timescale = int(update.message.text.replace('/daily', '').strip()) - except (TypeError, ValueError): + timescale = int(context.args[0]) + except (TypeError, ValueError, IndexError): timescale = 7 try: stats = self._rpc_daily_profit( @@ -272,12 +270,12 @@ class Telegram(RPC): ], tablefmt='simple') message = f'Daily Profit over the last {timescale} days:\n
{stats_tab}
' - self._send_msg(message, bot=bot, parse_mode=ParseMode.HTML) + self._send_msg(message, parse_mode=ParseMode.HTML) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _profit(self, bot: Bot, update: Update) -> None: + def _profit(self, update: Update, context: CallbackContext) -> None: """ Handler for /profit. Returns a cumulative profit statistics. @@ -317,12 +315,12 @@ class Telegram(RPC): f"*Latest Trade opened:* `{latest_trade_date}`\n" \ f"*Avg. Duration:* `{avg_duration}`\n" \ f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`" - self._send_msg(markdown_msg, bot=bot) + self._send_msg(markdown_msg) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _balance(self, bot: Bot, update: Update) -> None: + def _balance(self, update: Update, context: CallbackContext) -> None: """ Handler for /balance """ try: result = self._rpc_balance(self._config.get('fiat_display_currency', '')) @@ -330,16 +328,16 @@ class Telegram(RPC): for currency in result['currencies']: if currency['est_btc'] > 0.0001: curr_output = "*{currency}:*\n" \ - "\t`Available: {available: .8f}`\n" \ + "\t`Available: {free: .8f}`\n" \ "\t`Balance: {balance: .8f}`\n" \ - "\t`Pending: {pending: .8f}`\n" \ + "\t`Pending: {used: .8f}`\n" \ "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) else: curr_output = "*{currency}:* not showing <1$ amount \n".format(**currency) # Handle overflowing messsage length if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH: - self._send_msg(output, bot=bot) + self._send_msg(output) output = curr_output else: output += curr_output @@ -347,12 +345,12 @@ class Telegram(RPC): output += "\n*Estimated Value*:\n" \ "\t`BTC: {total: .8f}`\n" \ "\t`{symbol}: {value: .2f}`\n".format(**result) - self._send_msg(output, bot=bot) + self._send_msg(output) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _start(self, bot: Bot, update: Update) -> None: + def _start(self, update: Update, context: CallbackContext) -> None: """ Handler for /start. Starts TradeThread @@ -361,10 +359,10 @@ class Telegram(RPC): :return: None """ msg = self._rpc_start() - self._send_msg('Status: `{status}`'.format(**msg), bot=bot) + self._send_msg('Status: `{status}`'.format(**msg)) @authorized_only - def _stop(self, bot: Bot, update: Update) -> None: + def _stop(self, update: Update, context: CallbackContext) -> None: """ Handler for /stop. Stops TradeThread @@ -373,10 +371,10 @@ class Telegram(RPC): :return: None """ msg = self._rpc_stop() - self._send_msg('Status: `{status}`'.format(**msg), bot=bot) + self._send_msg('Status: `{status}`'.format(**msg)) @authorized_only - def _reload_conf(self, bot: Bot, update: Update) -> None: + def _reload_conf(self, update: Update, context: CallbackContext) -> None: """ Handler for /reload_conf. Triggers a config file reload @@ -385,10 +383,10 @@ class Telegram(RPC): :return: None """ msg = self._rpc_reload_conf() - self._send_msg('Status: `{status}`'.format(**msg), bot=bot) + self._send_msg('Status: `{status}`'.format(**msg)) @authorized_only - def _stopbuy(self, bot: Bot, update: Update) -> None: + def _stopbuy(self, update: Update, context: CallbackContext) -> None: """ Handler for /stop_buy. Sets max_open_trades to 0 and gracefully sells all open trades @@ -397,10 +395,10 @@ class Telegram(RPC): :return: None """ msg = self._rpc_stopbuy() - self._send_msg('Status: `{status}`'.format(**msg), bot=bot) + self._send_msg('Status: `{status}`'.format(**msg)) @authorized_only - def _forcesell(self, bot: Bot, update: Update) -> None: + def _forcesell(self, update: Update, context: CallbackContext) -> None: """ Handler for /forcesell . Sells the given trade at current price @@ -409,16 +407,16 @@ class Telegram(RPC): :return: None """ - trade_id = update.message.text.replace('/forcesell', '').strip() + trade_id = context.args[0] if len(context.args) > 0 else None try: msg = self._rpc_forcesell(trade_id) - self._send_msg('Forcesell Result: `{result}`'.format(**msg), bot=bot) + self._send_msg('Forcesell Result: `{result}`'.format(**msg)) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _forcebuy(self, bot: Bot, update: Update) -> None: + def _forcebuy(self, update: Update, context: CallbackContext) -> None: """ Handler for /forcebuy . Buys a pair trade at the given or current price @@ -427,16 +425,15 @@ class Telegram(RPC): :return: None """ - message = update.message.text.replace('/forcebuy', '').strip().split() - pair = message[0] - price = float(message[1]) if len(message) > 1 else None + pair = context.args[0] + price = float(context.args[1]) if len(context.args) > 1 else None try: self._rpc_forcebuy(pair, price) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _performance(self, bot: Bot, update: Update) -> None: + def _performance(self, update: Update, context: CallbackContext) -> None: """ Handler for /performance. Shows a performance statistic from finished trades @@ -455,10 +452,10 @@ class Telegram(RPC): message = 'Performance:\n{}'.format(stats) self._send_msg(message, parse_mode=ParseMode.HTML) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _count(self, bot: Bot, update: Update) -> None: + def _count(self, update: Update, context: CallbackContext) -> None: """ Handler for /count. Returns the number of trades running @@ -475,10 +472,10 @@ class Telegram(RPC): logger.debug(message) self._send_msg(message, parse_mode=ParseMode.HTML) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _whitelist(self, bot: Bot, update: Update) -> None: + def _whitelist(self, update: Update, context: CallbackContext) -> None: """ Handler for /whitelist Shows the currently active whitelist @@ -492,17 +489,17 @@ class Telegram(RPC): logger.debug(message) self._send_msg(message) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _blacklist(self, bot: Bot, update: Update, args: List[str]) -> None: + def _blacklist(self, update: Update, context: CallbackContext) -> None: """ Handler for /blacklist Shows the currently active blacklist """ try: - blacklist = self._rpc_blacklist(args) + blacklist = self._rpc_blacklist(context.args) message = f"Blacklist contains {blacklist['length']} pairs\n" message += f"`{', '.join(blacklist['blacklist'])}`" @@ -510,10 +507,10 @@ class Telegram(RPC): logger.debug(message) self._send_msg(message) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _edge(self, bot: Bot, update: Update) -> None: + def _edge(self, update: Update, context: CallbackContext) -> None: """ Handler for /edge Shows information related to Edge @@ -522,12 +519,12 @@ class Telegram(RPC): edge_pairs = self._rpc_edge() edge_pairs_tab = tabulate(edge_pairs, headers='keys', tablefmt='simple') message = f'Edge only validated following pairs:\n
{edge_pairs_tab}
' - self._send_msg(message, bot=bot, parse_mode=ParseMode.HTML) + self._send_msg(message, parse_mode=ParseMode.HTML) except RPCException as e: - self._send_msg(str(e), bot=bot) + self._send_msg(str(e)) @authorized_only - def _help(self, bot: Bot, update: Update) -> None: + def _help(self, update: Update, context: CallbackContext) -> None: """ Handler for /help. Show commands of the bot @@ -559,10 +556,10 @@ class Telegram(RPC): "*/help:* `This help message`\n" \ "*/version:* `Show version`" - self._send_msg(message, bot=bot) + self._send_msg(message) @authorized_only - def _version(self, bot: Bot, update: Update) -> None: + def _version(self, update: Update, context: CallbackContext) -> None: """ Handler for /version. Show version information @@ -570,10 +567,9 @@ class Telegram(RPC): :param update: message update :return: None """ - self._send_msg('*Version:* `{}`'.format(__version__), bot=bot) + self._send_msg('*Version:* `{}`'.format(__version__)) - def _send_msg(self, msg: str, bot: Bot = None, - parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: + def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: """ Send given markdown message :param msg: message @@ -581,7 +577,6 @@ class Telegram(RPC): :param parse_mode: telegram parse mode :return: None """ - bot = bot or self._updater.bot keyboard = [['/daily', '/profit', '/balance'], ['/status', '/status table', '/performance'], @@ -591,7 +586,7 @@ class Telegram(RPC): try: try: - bot.send_message( + self._updater.bot.send_message( self._config['telegram']['chat_id'], text=msg, parse_mode=parse_mode, @@ -604,7 +599,7 @@ class Telegram(RPC): 'Telegram NetworkError: %s! Trying one more time.', network_err.message ) - bot.send_message( + self._updater.bot.send_message( self._config['telegram']['chat_id'], text=msg, parse_mode=parse_mode, diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index bfc82b8d6..37ca466de 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -43,7 +43,9 @@ class Webhook(RPC): valuedict = self._config['webhook'].get('webhookbuy', None) elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: valuedict = self._config['webhook'].get('webhooksell', None) - elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: + elif msg['type'] in(RPCMessageType.STATUS_NOTIFICATION, + RPCMessageType.CUSTOM_NOTIFICATION, + RPCMessageType.WARNING_NOTIFICATION): valuedict = self._config['webhook'].get('webhookstatus', None) else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) diff --git a/freqtrade/state.py b/freqtrade/state.py index ce2683a77..d4a2adba0 100644 --- a/freqtrade/state.py +++ b/freqtrade/state.py @@ -25,4 +25,5 @@ class RunMode(Enum): BACKTEST = "backtest" EDGE = "edge" HYPEROPT = "hyperopt" + PLOT = "plot" OTHER = "other" # Used for plotting scripts and test diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index c62bfe5dc..40a4a0bea 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -1,45 +1 @@ -import logging -import sys -from copy import deepcopy - -from freqtrade.strategy.interface import IStrategy -# Import Default-Strategy to have hyperopt correctly resolve -from freqtrade.strategy.default_strategy import DefaultStrategy # noqa: F401 - - -logger = logging.getLogger(__name__) - - -def import_strategy(strategy: IStrategy, config: dict) -> IStrategy: - """ - Imports given Strategy instance to global scope - of freqtrade.strategy and returns an instance of it - """ - - # Copy all attributes from base class and class - comb = {**strategy.__class__.__dict__, **strategy.__dict__} - - # Delete '_abc_impl' from dict as deepcopy fails on 3.7 with - # `TypeError: can't pickle _abc_data objects`` - # This will only apply to python 3.7 - if sys.version_info.major == 3 and sys.version_info.minor == 7 and '_abc_impl' in comb: - del comb['_abc_impl'] - - attr = deepcopy(comb) - - # Adjust module name - attr['__module__'] = 'freqtrade.strategy' - - name = strategy.__class__.__name__ - clazz = type(name, (IStrategy,), attr) - - logger.debug( - 'Imported strategy %s.%s as %s.%s', - strategy.__module__, strategy.__class__.__name__, - clazz.__module__, strategy.__class__.__name__, - ) - - # Modify global scope to declare class - globals()[name] = clazz - - return clazz(config) +from freqtrade.strategy.interface import IStrategy # noqa: F401 diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 5c7d50a65..b839a9618 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -4,15 +4,18 @@ import talib.abstract as ta from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.indicator_helpers import fishers_inverse from freqtrade.strategy.interface import IStrategy class DefaultStrategy(IStrategy): """ Default Strategy provided by freqtrade bot. - You can override it with your own strategy + Please do not modify this strategy, it's intended for internal use only. + Please look at the SampleStrategy in the user_data/strategy directory + or strategy repository https://github.com/freqtrade/freqtrade-strategies + for samples and inspiration. """ + INTERFACE_VERSION = 2 # Minimal ROI designed for the strategy minimal_roi = { @@ -73,67 +76,25 @@ class DefaultStrategy(IStrategy): # ADX dataframe['adx'] = ta.ADX(dataframe) - # Awesome oscillator - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - """ - # Commodity Channel Index: values Oversold:<-100, Overbought:>100 - dataframe['cci'] = ta.CCI(dataframe) - """ # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] dataframe['macdhist'] = macd['macdhist'] - # MFI - dataframe['mfi'] = ta.MFI(dataframe) - # Minus Directional Indicator / Movement - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Plus Directional Indicator / Movement - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - """ - # ROC - dataframe['roc'] = ta.ROC(dataframe) - """ # RSI dataframe['rsi'] = ta.RSI(dataframe) - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi'] = fishers_inverse(dataframe['rsi']) - - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - # Stoch fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] dataframe['fastk'] = stoch_fast['fastk'] - """ - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] - """ - - # Overlap Studies - # ------------------------------------ - - # Previous Bollinger bands - # Because ta.BBANDS implementation is broken with small numbers, it actually - # returns middle band for all the three bands. Switch to qtpylib.bollinger_bands - # and use middle band instead. - dataframe['blower'] = ta.BBANDS(dataframe, nbdevup=2, nbdevdn=2)['lowerband'] # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) @@ -142,88 +103,11 @@ class DefaultStrategy(IStrategy): dataframe['bb_upperband'] = bollinger['upper'] # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - - # SAR Parabol - dataframe['sar'] = ta.SAR(dataframe) # SMA - Simple Moving Average dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - - # Cycle Indicator - # ------------------------------------ - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] - - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ - - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] - return dataframe def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 99f5f26de..17246ecf7 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -39,6 +39,7 @@ class SellType(Enum): TRAILING_STOP_LOSS = "trailing_stop_loss" SELL_SIGNAL = "sell_signal" FORCE_SELL = "force_sell" + EMERGENCY_SELL = "emergency_sell" NONE = "" @@ -60,6 +61,11 @@ class IStrategy(ABC): stoploss -> float: optimal stoploss designed for the strategy ticker_interval -> str: value of the ticker interval to use for the strategy """ + # Strategy interface version + # Default to version 2 + # Version 1 is the initial interface without metadata dict + # Version 2 populate_* include metadata dict + INTERFACE_VERSION: int = 2 _populate_fun_len: int = 0 _buy_fun_len: int = 0 @@ -196,7 +202,6 @@ class IStrategy(ABC): :param metadata: Metadata dictionary with additional data (e.g. 'pair') :return: DataFrame with ticker data and indicator data """ - pair = str(metadata.get('pair')) # Test if seen this pair and last candle before. @@ -286,7 +291,6 @@ class IStrategy(ABC): :param force_stoploss: Externally provided stoploss :return: True if trade should be sold, False otherwise """ - # Set current rate to low for backtesting sell current_rate = low or rate current_profit = trade.calc_profit_percent(current_rate) @@ -298,6 +302,8 @@ class IStrategy(ABC): force_stoploss=force_stoploss, high=high) if stoplossflag.sell_flag: + logger.debug(f"{trade.pair} - Stoploss hit. sell_flag=True, " + f"sell_type={stoplossflag.sell_type}") return stoplossflag # Set current rate to high for backtesting sell @@ -306,22 +312,31 @@ class IStrategy(ABC): experimental = self.config.get('experimental', {}) if buy and experimental.get('ignore_roi_if_buy_signal', False): - logger.debug('Buy signal still active - not selling.') + # This one is noisy, commented out + # logger.debug(f"{trade.pair} - Buy signal still active. sell_flag=False") return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): - logger.debug('Required profit reached. Selling..') + logger.debug(f"{trade.pair} - Required profit reached. sell_flag=True, " + f"sell_type=SellType.ROI") return SellCheckTuple(sell_flag=True, sell_type=SellType.ROI) if experimental.get('sell_profit_only', False): - logger.debug('Checking if trade is profitable..') + # This one is noisy, commented out + # logger.debug(f"{trade.pair} - Checking if trade is profitable...") if trade.calc_profit(rate=rate) <= 0: + # This one is noisy, commented out + # logger.debug(f"{trade.pair} - Trade is not profitable. sell_flag=False") return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) + if sell and not buy and experimental.get('use_sell_signal', False): - logger.debug('Sell signal received. Selling..') + logger.debug(f"{trade.pair} - Sell signal received. sell_flag=True, " + f"sell_type=SellType.SELL_SIGNAL") return SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL) + # This one is noisy, commented out... + # logger.debug(f"{trade.pair} - No sell signal. sell_flag=False") return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) def stop_loss_reached(self, current_rate: float, trade: Trade, @@ -332,7 +347,6 @@ class IStrategy(ABC): decides to sell or not :param current_profit: current profit in percent """ - trailing_stop = self.config.get('trailing_stop', False) stop_loss_value = force_stoploss if force_stoploss else self.stoploss @@ -353,7 +367,7 @@ class IStrategy(ABC): if 'trailing_stop_positive' in self.config and high_profit > sl_offset: # Ignore mypy error check in configuration that this is a float stop_loss_value = self.config.get('trailing_stop_positive') # type: ignore - logger.debug(f"using positive stop loss: {stop_loss_value} " + logger.debug(f"{trade.pair} - Using positive stoploss: {stop_loss_value} " f"offset: {sl_offset:.4g} profit: {current_profit:.4f}%") trade.adjust_stop_loss(high or current_rate, stop_loss_value) @@ -363,20 +377,20 @@ class IStrategy(ABC): (trade.stop_loss >= current_rate) and (not self.order_types.get('stoploss_on_exchange'))): - selltype = SellType.STOP_LOSS + sell_type = SellType.STOP_LOSS # If initial stoploss is not the same as current one then it is trailing. if trade.initial_stop_loss != trade.stop_loss: - selltype = SellType.TRAILING_STOP_LOSS + sell_type = SellType.TRAILING_STOP_LOSS logger.debug( - f"HIT STOP: current price at {current_rate:.6f}, " - f"stop loss is {trade.stop_loss:.6f}, " - f"initial stop loss was at {trade.initial_stop_loss:.6f}, " + f"{trade.pair} - HIT STOP: current price at {current_rate:.6f}, " + f"stoploss is {trade.stop_loss:.6f}, " + f"initial stoploss was at {trade.initial_stop_loss:.6f}, " f"trade opened at {trade.open_rate:.6f}") - logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}") + logger.debug(f"{trade.pair} - Trailing stop saved " + f"{trade.stop_loss - trade.initial_stop_loss:.6f}") - logger.debug('Stop loss hit.') - return SellCheckTuple(sell_flag=True, sell_type=selltype) + return SellCheckTuple(sell_flag=True, sell_type=sell_type) return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) diff --git a/freqtrade/tests/exchange/test_kraken.py b/freqtrade/tests/exchange/test_kraken.py deleted file mode 100644 index 8f476affb..000000000 --- a/freqtrade/tests/exchange/test_kraken.py +++ /dev/null @@ -1,69 +0,0 @@ -# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement -# pragma pylint: disable=protected-access -from random import randint -from unittest.mock import MagicMock - -from freqtrade.tests.conftest import get_patched_exchange - - -def test_buy_kraken_trading_agreement(default_conf, mocker): - api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - order_type = 'limit' - time_in_force = 'ioc' - api_mock.options = {} - api_mock.create_order = MagicMock(return_value={ - 'id': order_id, - 'info': { - 'foo': 'bar' - } - }) - default_conf['dry_run'] = False - - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") - - order = exchange.buy(pair='ETH/BTC', ordertype=order_type, - amount=1, rate=200, time_in_force=time_in_force) - - assert 'id' in order - assert 'info' in order - assert order['id'] == order_id - assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' - assert api_mock.create_order.call_args[0][1] == order_type - assert api_mock.create_order.call_args[0][2] == 'buy' - assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][4] == 200 - assert api_mock.create_order.call_args[0][5] == {'timeInForce': 'ioc', - 'trading_agreement': 'agree'} - - -def test_sell_kraken_trading_agreement(default_conf, mocker): - api_mock = MagicMock() - order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) - order_type = 'market' - api_mock.options = {} - api_mock.create_order = MagicMock(return_value={ - 'id': order_id, - 'info': { - 'foo': 'bar' - } - }) - default_conf['dry_run'] = False - - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") - - order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) - - assert 'id' in order - assert 'info' in order - assert order['id'] == order_id - assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' - assert api_mock.create_order.call_args[0][1] == order_type - assert api_mock.create_order.call_args[0][2] == 'sell' - assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][4] is None - assert api_mock.create_order.call_args[0][5] == {'trading_agreement': 'agree'} diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py deleted file mode 100644 index 24f11e32e..000000000 --- a/freqtrade/tests/test_arguments.py +++ /dev/null @@ -1,181 +0,0 @@ -# pragma pylint: disable=missing-docstring, C0103 -import argparse - -import pytest - -from freqtrade.configuration import Arguments -from freqtrade.configuration.arguments import ARGS_PLOT_DATAFRAME -from freqtrade.configuration.cli_options import check_int_positive - - -# Parse common command-line-arguments. Used for all tools -def test_parse_args_none() -> None: - arguments = Arguments([], '') - assert isinstance(arguments, Arguments) - assert isinstance(arguments.parser, argparse.ArgumentParser) - - -def test_parse_args_defaults() -> None: - args = Arguments([], '').get_parsed_arg() - assert args.config == ['config.json'] - assert args.strategy_path is None - assert args.datadir is None - assert args.verbosity == 0 - - -def test_parse_args_config() -> None: - args = Arguments(['-c', '/dev/null'], '').get_parsed_arg() - assert args.config == ['/dev/null'] - - args = Arguments(['--config', '/dev/null'], '').get_parsed_arg() - assert args.config == ['/dev/null'] - - args = Arguments(['--config', '/dev/null', - '--config', '/dev/zero'], - '').get_parsed_arg() - assert args.config == ['/dev/null', '/dev/zero'] - - -def test_parse_args_db_url() -> None: - args = Arguments(['--db-url', 'sqlite:///test.sqlite'], '').get_parsed_arg() - assert args.db_url == 'sqlite:///test.sqlite' - - -def test_parse_args_verbose() -> None: - args = Arguments(['-v'], '').get_parsed_arg() - assert args.verbosity == 1 - - args = Arguments(['--verbose'], '').get_parsed_arg() - assert args.verbosity == 1 - - -def test_common_scripts_options() -> None: - args = Arguments(['download-data', '-p', 'ETH/BTC', 'XRP/BTC'], '').get_parsed_arg() - - assert args.pairs == ['ETH/BTC', 'XRP/BTC'] - assert hasattr(args, "func") - - -def test_parse_args_version() -> None: - with pytest.raises(SystemExit, match=r'0'): - Arguments(['--version'], '').get_parsed_arg() - - -def test_parse_args_invalid() -> None: - with pytest.raises(SystemExit, match=r'2'): - Arguments(['-c'], '').get_parsed_arg() - - -def test_parse_args_strategy() -> None: - args = Arguments(['--strategy', 'SomeStrategy'], '').get_parsed_arg() - assert args.strategy == 'SomeStrategy' - - -def test_parse_args_strategy_invalid() -> None: - with pytest.raises(SystemExit, match=r'2'): - Arguments(['--strategy'], '').get_parsed_arg() - - -def test_parse_args_strategy_path() -> None: - args = Arguments(['--strategy-path', '/some/path'], '').get_parsed_arg() - assert args.strategy_path == '/some/path' - - -def test_parse_args_strategy_path_invalid() -> None: - with pytest.raises(SystemExit, match=r'2'): - Arguments(['--strategy-path'], '').get_parsed_arg() - - -def test_parse_args_backtesting_invalid() -> None: - with pytest.raises(SystemExit, match=r'2'): - Arguments(['backtesting --ticker-interval'], '').get_parsed_arg() - - with pytest.raises(SystemExit, match=r'2'): - Arguments(['backtesting --ticker-interval', 'abc'], '').get_parsed_arg() - - -def test_parse_args_backtesting_custom() -> None: - args = [ - '-c', 'test_conf.json', - 'backtesting', - '--ticker-interval', '1m', - '--refresh-pairs-cached', - '--strategy-list', - 'DefaultStrategy', - 'TestStrategy' - ] - call_args = Arguments(args, '').get_parsed_arg() - assert call_args.config == ['test_conf.json'] - assert call_args.verbosity == 0 - assert call_args.subparser == 'backtesting' - assert call_args.func is not None - assert call_args.ticker_interval == '1m' - assert call_args.refresh_pairs is True - assert type(call_args.strategy_list) is list - assert len(call_args.strategy_list) == 2 - - -def test_parse_args_hyperopt_custom() -> None: - args = [ - '-c', 'test_conf.json', - 'hyperopt', - '--epochs', '20', - '--spaces', 'buy' - ] - call_args = Arguments(args, '').get_parsed_arg() - assert call_args.config == ['test_conf.json'] - assert call_args.epochs == 20 - assert call_args.verbosity == 0 - assert call_args.subparser == 'hyperopt' - assert call_args.spaces == ['buy'] - assert call_args.func is not None - - -def test_download_data_options() -> None: - args = [ - '--datadir', 'datadir/directory', - 'download-data', - '--pairs-file', 'file_with_pairs', - '--days', '30', - '--exchange', 'binance' - ] - args = Arguments(args, '').get_parsed_arg() - - assert args.pairs_file == 'file_with_pairs' - assert args.datadir == 'datadir/directory' - assert args.days == 30 - assert args.exchange == 'binance' - - -def test_plot_dataframe_options() -> None: - args = [ - '--indicators1', 'sma10,sma100', - '--indicators2', 'macd,fastd,fastk', - '--plot-limit', '30', - '-p', 'UNITTEST/BTC', - ] - arguments = Arguments(args, '') - arguments._build_args(ARGS_PLOT_DATAFRAME) - pargs = arguments._parse_args() - assert pargs.indicators1 == "sma10,sma100" - assert pargs.indicators2 == "macd,fastd,fastk" - assert pargs.plot_limit == 30 - assert pargs.pairs == ["UNITTEST/BTC"] - - -def test_check_int_positive() -> None: - assert check_int_positive("3") == 3 - assert check_int_positive("1") == 1 - assert check_int_positive("100") == 100 - - with pytest.raises(argparse.ArgumentTypeError): - check_int_positive("-2") - - with pytest.raises(argparse.ArgumentTypeError): - check_int_positive("0") - - with pytest.raises(argparse.ArgumentTypeError): - check_int_positive("3.5") - - with pytest.raises(argparse.ArgumentTypeError): - check_int_positive("DeadBeef") diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 56e60ec82..6ce5e888c 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -1,14 +1,14 @@ import logging import sys -from argparse import Namespace from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, List import arrow +from freqtrade import OperationalException from freqtrade.configuration import Configuration, TimeRange from freqtrade.configuration.directory_operations import create_userdata_dir -from freqtrade.data.history import download_pair_history +from freqtrade.data.history import refresh_backtest_ohlcv_data from freqtrade.exchange import available_exchanges from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode @@ -16,7 +16,7 @@ from freqtrade.state import RunMode logger = logging.getLogger(__name__) -def setup_utils_configuration(args: Namespace, method: RunMode) -> Dict[str, Any]: +def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: """ Prepare the configuration for utils subcommands :param args: Cli args from Arguments() @@ -33,34 +33,34 @@ def setup_utils_configuration(args: Namespace, method: RunMode) -> Dict[str, Any return config -def start_list_exchanges(args: Namespace) -> None: +def start_list_exchanges(args: Dict[str, Any]) -> None: """ Print available exchanges :param args: Cli args from Arguments() :return: None """ - if args.print_one_column: + if args['print_one_column']: print('\n'.join(available_exchanges())) else: print(f"Exchanges supported by ccxt and available for Freqtrade: " f"{', '.join(available_exchanges())}") -def start_create_userdir(args: Namespace) -> None: +def start_create_userdir(args: Dict[str, Any]) -> None: """ Create "user_data" directory to contain user data strategies, hyperopts, ...) :param args: Cli args from Arguments() :return: None """ - if "user_data_dir" in args and args.user_data_dir: - create_userdata_dir(args.user_data_dir, create_dir=True) + if "user_data_dir" in args and args["user_data_dir"]: + create_userdata_dir(args["user_data_dir"], create_dir=True) else: logger.warning("`create-userdir` requires --userdir to be set.") sys.exit(1) -def start_download_data(args: Namespace) -> None: +def start_download_data(args: Dict[str, Any]) -> None: """ Download data (former download_backtest_data.py script) """ @@ -71,43 +71,29 @@ def start_download_data(args: Namespace) -> None: time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") timerange = TimeRange.parse_timerange(f'{time_since}-') + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + dl_path = Path(config['datadir']) logger.info(f'About to download pairs: {config["pairs"]}, ' f'intervals: {config["timeframes"]} to {dl_path}') - pairs_not_available = [] + pairs_not_available: List[str] = [] try: # Init exchange exchange = ExchangeResolver(config['exchange']['name'], config).exchange - for pair in config["pairs"]: - if pair not in exchange.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - for ticker_interval in config["timeframes"]: - pair_print = pair.replace('/', '_') - filename = f'{pair_print}-{ticker_interval}.json' - dl_file = dl_path.joinpath(filename) - if config.get("erase") and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}, interval {ticker_interval}.') - dl_file.unlink() - - logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') - download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, ticker_interval=str(ticker_interval), - timerange=timerange) + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=config["pairs"], timeframes=config["timeframes"], + dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") finally: if pairs_not_available: - logger.info( - f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {config['exchange']['name']}.") - - # configuration.resolve_pairs_list() - print(config) + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {config['exchange']['name']}.") diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index c8ab90276..90b68c49d 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -17,7 +17,7 @@ class Wallet(NamedTuple): total: float = 0 -class Wallets(object): +class Wallets: def __init__(self, config: dict, exchange: Exchange) -> None: self._config = config diff --git a/freqtrade/worker.py b/freqtrade/worker.py index df792e35e..8e4be9d43 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -4,27 +4,26 @@ Main Freqtrade worker class. import logging import time import traceback -from argparse import Namespace -from typing import Any, Callable, Optional +from typing import Any, Callable, Dict, Optional + import sdnotify -from freqtrade import (constants, OperationalException, TemporaryError, - __version__) +from freqtrade import (OperationalException, TemporaryError, __version__, + constants) from freqtrade.configuration import Configuration from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.state import State from freqtrade.rpc import RPCMessageType - +from freqtrade.state import State logger = logging.getLogger(__name__) -class Worker(object): +class Worker: """ Freqtradebot worker class """ - def __init__(self, args: Namespace, config=None) -> None: + def __init__(self, args: Dict[str, Any], config=None) -> None: """ Init all variables and objects the bot needs to work """ diff --git a/mkdocs.yml b/mkdocs.yml index b5e759432..7aedb4bba 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,15 +11,16 @@ nav: - Telegram: telegram-usage.md - Web Hook: webhook-config.md - REST API: rest-api.md + - Data Downloading: data-download.md - Backtesting: backtesting.md - Hyperopt: hyperopt.md - Edge positioning: edge.md - - Plotting: plotting.md - - Deprecated features: deprecated.md - FAQ: faq.md - Data Analysis: data-analysis.md + - Plotting: plotting.md - SQL Cheatsheet: sql_cheatsheet.md - Sandbox testing: sandbox-testing.md + - Deprecated features: deprecated.md - Contributors guide: developer.md theme: name: material diff --git a/requirements-common.txt b/requirements-common.txt index 3d80c3ef5..f10134203 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,28 +1,22 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.1068 -SQLAlchemy==1.3.7 -python-telegram-bot==11.1.0 -arrow==0.14.5 +ccxt==1.18.1180 +SQLAlchemy==1.3.8 +python-telegram-bot==12.1.1 +arrow==0.15.2 cachetools==3.1.1 requests==2.22.0 -urllib3==1.25.3 +urllib3==1.25.5 wrapt==1.11.2 -scikit-learn==0.21.3 -joblib==0.13.2 jsonschema==3.0.2 TA-Lib==0.4.17 tabulate==0.8.3 coinmarketcap==5.0.3 -# Required for hyperopt -scikit-optimize==0.5.2 -filelock==3.0.12 - # find first, C search in arrays py_find_1st==1.1.4 -#Load ticker files 30% faster +# Load ticker files 30% faster python-rapidjson==0.8.0 # Notify systemd diff --git a/requirements-dev.txt b/requirements-dev.txt index 6436c60e4..dcf2c7217 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,13 +1,14 @@ # Include all requirements to run the bot. -r requirements.txt -r requirements-plot.txt +-r requirements-hyperopt.txt coveralls==1.8.2 flake8==3.7.8 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 mypy==0.720 -pytest==5.1.0 +pytest==5.1.3 pytest-asyncio==0.10.0 pytest-cov==2.7.1 pytest-mock==1.10.4 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt new file mode 100644 index 000000000..bb0ad60f0 --- /dev/null +++ b/requirements-hyperopt.txt @@ -0,0 +1,9 @@ +# Include all requirements to run the bot. +# -r requirements.txt + +# Required for hyperopt +scipy==1.3.1 +scikit-learn==0.21.3 +scikit-optimize==0.5.2 +filelock==3.0.12 +joblib==0.13.2 diff --git a/requirements-plot.txt b/requirements-plot.txt index f10bfac3f..1f1df4ecc 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.1.0 +plotly==4.1.1 diff --git a/requirements.txt b/requirements.txt index 9d558b5b8..2767180ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ # Load common requirements -r requirements-common.txt -numpy==1.17.0 -pandas==0.25.0 -scipy==1.3.1 +numpy==1.17.2 +pandas==0.25.1 diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index db4f99d61..62c4bc39f 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -1,100 +1,11 @@ #!/usr/bin/env python3 -""" -Script to display when the bot will buy on specific pair(s) -Use `python plot_dataframe.py --help` to display the command line arguments - -Indicators recommended -Row 1: sma, ema3, ema5, ema10, ema50 -Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk - -Example of usage: -> python3 scripts/plot_dataframe.py --pairs BTC/EUR,XRP/BTC -d user_data/data/ - --indicators1 sma,ema3 --indicators2 fastk,fastd -""" -import logging import sys -from typing import Any, Dict, List - -from freqtrade.configuration import Arguments -from freqtrade.configuration.arguments import ARGS_PLOT_DATAFRAME -from freqtrade.data.btanalysis import extract_trades_of_period -from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import (init_plotscript, generate_candlestick_graph, - store_plot_file, - generate_plot_filename) -from freqtrade.state import RunMode - -logger = logging.getLogger(__name__) -def analyse_and_plot_pairs(config: Dict[str, Any]): - """ - From arguments provided in cli: - -Initialise backtest env - -Get tickers data - -Generate Dafaframes populated with indicators and signals - -Load trades excecuted on same periods - -Generate Plotly plot objects - -Generate plot files - :return: None - """ - plot_elements = init_plotscript(config) - trades = plot_elements['trades'] - strategy = plot_elements["strategy"] +print("This script has been integrated into freqtrade " + "and its functionality is available by calling `freqtrade plot-dataframe`.") +print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ " + "for details.") - pair_counter = 0 - for pair, data in plot_elements["tickers"].items(): - pair_counter += 1 - logger.info("analyse pair %s", pair) - tickers = {} - tickers[pair] = data - - dataframe = strategy.analyze_ticker(tickers[pair], {'pair': pair}) - - trades_pair = trades.loc[trades['pair'] == pair] - trades_pair = extract_trades_of_period(dataframe, trades_pair) - - fig = generate_candlestick_graph( - pair=pair, - data=dataframe, - trades=trades_pair, - indicators1=config["indicators1"].split(","), - indicators2=config["indicators2"].split(",") - ) - - store_plot_file(fig, filename=generate_plot_filename(pair, config['ticker_interval']), - directory=config['user_data_dir'] / "plot") - - logger.info('End of ploting process %s plots generated', pair_counter) - - -def plot_parse_args(args: List[str]) -> Dict[str, Any]: - """ - Parse args passed to the script - :param args: Cli arguments - :return: args: Array with all arguments - """ - arguments = Arguments(args, 'Graph dataframe') - arguments._build_args(optionlist=ARGS_PLOT_DATAFRAME) - parsed_args = arguments._parse_args() - - # Load the configuration - config = setup_configuration(parsed_args, RunMode.OTHER) - return config - - -def main(sysargv: List[str]) -> None: - """ - This function will initiate the bot and start the trading loop. - :return: None - """ - logger.info('Starting Plot Dataframe') - analyse_and_plot_pairs( - plot_parse_args(sysargv) - ) - exit() - - -if __name__ == '__main__': - main(sys.argv[1:]) +sys.exit(1) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 578ddf15f..c9a23c1ee 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -1,66 +1,11 @@ #!/usr/bin/env python3 -""" -Script to display profits -Use `python plot_profit.py --help` to display the command line arguments -""" -import logging import sys -from typing import Any, Dict, List - -from freqtrade.configuration import Arguments -from freqtrade.configuration.arguments import ARGS_PLOT_PROFIT -from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import init_plotscript, generate_profit_graph, store_plot_file -from freqtrade.state import RunMode - -logger = logging.getLogger(__name__) -def plot_profit(config: Dict[str, Any]) -> None: - """ - Plots the total profit for all pairs. - Note, the profit calculation isn't realistic. - But should be somewhat proportional, and therefor useful - in helping out to find a good algorithm. - """ - plot_elements = init_plotscript(config) - trades = plot_elements['trades'] - # Filter trades to relevant pairs - trades = trades[trades['pair'].isin(plot_elements["pairs"])] +print("This script has been integrated into freqtrade " + "and its functionality is available by calling `freqtrade plot-profit`.") +print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ " + "for details.") - # Create an average close price of all the pairs that were involved. - # this could be useful to gauge the overall market trend - fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades) - store_plot_file(fig, filename='freqtrade-profit-plot.html', - directory=config['user_data_dir'] / "plot", auto_open=True) - - -def plot_parse_args(args: List[str]) -> Dict[str, Any]: - """ - Parse args passed to the script - :param args: Cli arguments - :return: args: Array with all arguments - """ - arguments = Arguments(args, 'Graph profits') - arguments._build_args(optionlist=ARGS_PLOT_PROFIT) - parsed_args = arguments._parse_args() - - # Load the configuration - config = setup_configuration(parsed_args, RunMode.OTHER) - return config - - -def main(sysargv: List[str]) -> None: - """ - This function will initiate the bot and start the trading loop. - :return: None - """ - logger.info('Starting Plot Dataframe') - plot_profit( - plot_parse_args(sysargv) - ) - - -if __name__ == '__main__': - main(sys.argv[1:]) +sys.exit(1) diff --git a/setup.cfg b/setup.cfg index 473f50639..34f25482b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,9 +2,14 @@ #ignore = max-line-length = 100 max-complexity = 12 +exclude = + .git, + __pycache__, + .eggs, + user_data, [mypy] ignore_missing_imports = True -[mypy-freqtrade.tests.*] +[mypy-tests.*] ignore_errors = True diff --git a/setup.py b/setup.py index 631c8b654..2eb09e589 100644 --- a/setup.py +++ b/setup.py @@ -6,11 +6,25 @@ if version_info.major == 3 and version_info.minor < 6 or \ print('Your Python interpreter must be 3.6 or greater!') exit(1) -from freqtrade import __version__ +from pathlib import Path # noqa: E402 +from freqtrade import __version__ # noqa: E402 + + +readme_file = Path(__file__).parent / "README.md" +readme_long = "Crypto Trading Bot" +if readme_file.is_file(): + readme_long = (Path(__file__).parent / "README.md").read_text() # Requirements used for submodules api = ['flask'] plot = ['plotly>=4.0'] +hyperopt = [ + 'scipy', + 'scikit-learn', + 'scikit-optimize', + 'filelock', + 'joblib', + ] develop = [ 'coveralls', @@ -31,13 +45,15 @@ jupyter = [ 'ipykernel', ] -all_extra = api + plot + develop + jupyter +all_extra = api + plot + develop + jupyter + hyperopt setup(name='freqtrade', version=__version__, description='Crypto Trading Bot', + long_description=readme_long, + long_description_content_type="text/markdown", url='https://github.com/freqtrade/freqtrade', - author='gcarq and contributors', + author='Freqtrade Team', author_email='michael.egger@tsn.at', license='GPLv3', packages=['freqtrade'], @@ -45,7 +61,7 @@ setup(name='freqtrade', tests_require=['pytest', 'pytest-mock', 'pytest-cov'], install_requires=[ # from requirements-common.txt - 'ccxt>=1.18', + 'ccxt>=1.18.1080', 'SQLAlchemy', 'python-telegram-bot', 'arrow', @@ -53,14 +69,10 @@ setup(name='freqtrade', 'requests', 'urllib3', 'wrapt', - 'scikit-learn', - 'joblib', 'jsonschema', 'TA-Lib', 'tabulate', 'coinmarketcap', - 'scikit-optimize', - 'filelock', 'py_find_1st', 'python-rapidjson', 'sdnotify', @@ -68,15 +80,14 @@ setup(name='freqtrade', # from requirements.txt 'numpy', 'pandas', - 'scipy', ], extras_require={ 'api': api, 'dev': all_extra, 'plot': plot, - 'all': all_extra, 'jupyter': jupyter, - + 'hyperopt': hyperopt, + 'all': all_extra, }, include_package_data=True, zip_safe=False, diff --git a/freqtrade/tests/__init__.py b/tests/__init__.py similarity index 100% rename from freqtrade/tests/__init__.py rename to tests/__init__.py diff --git a/freqtrade/tests/config_test_comments.json b/tests/config_test_comments.json similarity index 100% rename from freqtrade/tests/config_test_comments.json rename to tests/config_test_comments.json diff --git a/freqtrade/tests/conftest.py b/tests/conftest.py similarity index 96% rename from freqtrade/tests/conftest.py rename to tests/conftest.py index 654b959ca..6a0a74b5b 100644 --- a/freqtrade/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,7 @@ def log_has_re(line, logs): def get_args(args): - return Arguments(args, '').get_parsed_arg() + return Arguments(args).get_parsed_arg() def patched_configuration_load_config_file(mocker, config) -> None: @@ -117,7 +117,7 @@ def patch_freqtradebot(mocker, config) -> None: """ mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) persistence.init(config['db_url']) - patch_exchange(mocker, None) + patch_exchange(mocker) mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) @@ -182,7 +182,7 @@ def init_persistence(default_conf): @pytest.fixture(scope="function") -def default_conf(): +def default_conf(testdatadir): """ Returns validated configuration suitable for most tests """ configuration = { "max_open_trades": 1, @@ -237,6 +237,7 @@ def default_conf(): "token": "token", "chat_id": "0" }, + "datadir": str(testdatadir), "initial_state": "running", "db_url": "sqlite://", "user_data_dir": Path("user_data"), @@ -890,8 +891,8 @@ def tickers(): @pytest.fixture -def result(): - with Path('freqtrade/tests/testdata/UNITTEST_BTC-1m.json').open('r') as data_file: +def result(testdatadir): + with (testdatadir / 'UNITTEST_BTC-1m.json').open('r') as data_file: return parse_ticker_dataframe(json.load(data_file), '1m', pair="UNITTEST/BTC", fill_missing=True) @@ -1047,3 +1048,30 @@ def rpc_balance(): 'used': 0.0 }, } + + +@pytest.fixture +def testdatadir() -> Path: + """Return the path where testdata files are stored""" + return (Path(__file__).parent / "testdata").resolve() + + +@pytest.fixture(scope="function") +def import_fails() -> None: + # Source of this test-method: + # https://stackoverflow.com/questions/2481511/mocking-importerror-in-python + import builtins + realimport = builtins.__import__ + + def mockedimport(name, *args, **kwargs): + if name in ["filelock"]: + raise ImportError(f"No module named '{name}'") + return realimport(name, *args, **kwargs) + + builtins.__import__ = mockedimport + + # Run test - then cleanup + yield + + # restore previous importfunction + builtins.__import__ = realimport diff --git a/freqtrade/tests/data/__init__.py b/tests/data/__init__.py similarity index 100% rename from freqtrade/tests/data/__init__.py rename to tests/data/__init__.py diff --git a/freqtrade/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py similarity index 83% rename from freqtrade/tests/data/test_btanalysis.py rename to tests/data/test_btanalysis.py index cf8cae566..18326226c 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -11,14 +11,13 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, extract_trades_of_period, load_backtest_data, load_trades, load_trades_from_db) -from freqtrade.data.history import (load_data, load_pair_history, - make_testdata_path) -from freqtrade.tests.test_persistence import create_mock_trades +from freqtrade.data.history import load_data, load_pair_history +from tests.test_persistence import create_mock_trades -def test_load_backtest_data(): +def test_load_backtest_data(testdatadir): - filename = make_testdata_path(None) / "backtest-result_test.json" + filename = testdatadir / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"] @@ -52,12 +51,12 @@ def test_load_trades_db(default_conf, fee, mocker): assert col in trades.columns -def test_extract_trades_of_period(): +def test_extract_trades_of_period(testdatadir): pair = "UNITTEST/BTC" timerange = TimeRange(None, 'line', 0, -1000) data = load_pair_history(pair=pair, ticker_interval='1m', - datadir=None, timerange=timerange) + datadir=testdatadir, timerange=timerange) # timerange = 2017-11-14 06:07 - 2017-11-14 22:58:00 trades = DataFrame( @@ -89,25 +88,28 @@ def test_load_trades(default_conf, mocker): db_mock = mocker.patch("freqtrade.data.btanalysis.load_trades_from_db", MagicMock()) bt_mock = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock()) - default_conf['trade_source'] = "DB" - load_trades(default_conf) + load_trades("DB", + db_url=default_conf.get('db_url'), + exportfilename=default_conf.get('exportfilename'), + ) assert db_mock.call_count == 1 assert bt_mock.call_count == 0 db_mock.reset_mock() bt_mock.reset_mock() - default_conf['trade_source'] = "file" default_conf['exportfilename'] = "testfile.json" - load_trades(default_conf) + load_trades("file", + db_url=default_conf.get('db_url'), + exportfilename=default_conf.get('exportfilename'),) assert db_mock.call_count == 0 assert bt_mock.call_count == 1 -def test_combine_tickers_with_mean(): +def test_combine_tickers_with_mean(testdatadir): pairs = ["ETH/BTC", "XLM/BTC"] - tickers = load_data(datadir=None, + tickers = load_data(datadir=testdatadir, pairs=pairs, ticker_interval='5m' ) @@ -118,13 +120,13 @@ def test_combine_tickers_with_mean(): assert "mean" in df.columns -def test_create_cum_profit(): - filename = make_testdata_path(None) / "backtest-result_test.json" +def test_create_cum_profit(testdatadir): + filename = testdatadir / "backtest-result_test.json" bt_data = load_backtest_data(filename) timerange = TimeRange.parse_timerange("20180110-20180112") df = load_pair_history(pair="POWR/BTC", ticker_interval='5m', - datadir=None, timerange=timerange) + datadir=testdatadir, timerange=timerange) cum_profits = create_cum_profit(df.set_index('date'), bt_data[bt_data["pair"] == 'POWR/BTC'], diff --git a/freqtrade/tests/data/test_converter.py b/tests/data/test_converter.py similarity index 96% rename from freqtrade/tests/data/test_converter.py rename to tests/data/test_converter.py index 39462bdd8..e773a970e 100644 --- a/freqtrade/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -3,7 +3,7 @@ import logging from freqtrade.data.converter import parse_ticker_dataframe, ohlcv_fill_up_missing_data from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timeframe -from freqtrade.tests.conftest import log_has +from tests.conftest import log_has def test_dataframe_correct_columns(result): @@ -21,10 +21,9 @@ def test_parse_ticker_dataframe(ticker_history_list, caplog): assert log_has('Parsing tickerlist to dataframe', caplog) -def test_ohlcv_fill_up_missing_data(caplog): - data = load_pair_history(datadir=None, +def test_ohlcv_fill_up_missing_data(testdatadir, caplog): + data = load_pair_history(datadir=testdatadir, ticker_interval='1m', - refresh_pairs=False, pair='UNITTEST/BTC', fill_up_missing=False) caplog.set_level(logging.DEBUG) diff --git a/freqtrade/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py similarity index 96% rename from freqtrade/tests/data/test_dataprovider.py rename to tests/data/test_dataprovider.py index 2272f69a3..39e2f7d2e 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -4,7 +4,7 @@ from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider from freqtrade.state import RunMode -from freqtrade.tests.conftest import get_patched_exchange +from tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ticker_history): @@ -45,8 +45,6 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history): data = dp.historic_ohlcv("UNITTEST/BTC", "5m") assert isinstance(data, DataFrame) assert historymock.call_count == 1 - assert historymock.call_args_list[0][1]["datadir"] is None - assert historymock.call_args_list[0][1]["refresh_pairs"] is False assert historymock.call_args_list[0][1]["ticker_interval"] == "5m" diff --git a/freqtrade/tests/data/test_history.py b/tests/data/test_history.py similarity index 80% rename from freqtrade/tests/data/test_history.py rename to tests/data/test_history.py index 7360f3c1c..e386c3506 100644 --- a/freqtrade/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -5,7 +5,7 @@ import os import uuid from pathlib import Path from shutil import copyfile -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import arrow import pytest @@ -16,13 +16,13 @@ from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, - load_tickerdata_file, make_testdata_path, + load_tickerdata_file, + refresh_backtest_ohlcv_data, trim_tickerlist) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.strategy.default_strategy import DefaultStrategy -from freqtrade.tests.conftest import (get_patched_exchange, log_has, - patch_exchange) +from tests.conftest import get_patched_exchange, log_has, log_has_re, patch_exchange # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -59,8 +59,8 @@ def _clean_test_file(file: str) -> None: os.rename(file_swp, file) -def test_load_data_30min_ticker(mocker, caplog, default_conf) -> None: - ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='30m', datadir=None) +def test_load_data_30min_ticker(mocker, caplog, default_conf, testdatadir) -> None: + ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='30m', datadir=testdatadir) assert isinstance(ld, DataFrame) assert not log_has( 'Download history data for pair: "UNITTEST/BTC", interval: 30m ' @@ -68,22 +68,21 @@ def test_load_data_30min_ticker(mocker, caplog, default_conf) -> None: ) -def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: - ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='7m', datadir=None) +def test_load_data_7min_ticker(mocker, caplog, default_conf, testdatadir) -> None: + ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='7m', datadir=testdatadir) assert not isinstance(ld, DataFrame) assert ld is None assert log_has( 'No history data for pair: "UNITTEST/BTC", interval: 7m. ' - 'Use --refresh-pairs-cached option or `freqtrade download-data` ' - 'script to download the data', caplog + 'Use `freqtrade download-data` to download the data', caplog ) -def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: +def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) - history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC']) + history.load_data(datadir=testdatadir, ticker_interval='1m', pairs=['UNITTEST/BTC']) assert os.path.isfile(file) is True assert not log_has( 'Download history data for pair: "UNITTEST/BTC", interval: 1m ' @@ -92,7 +91,8 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: _clean_test_file(file) -def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, default_conf) -> None: +def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, + default_conf, testdatadir) -> None: """ Test load_pair_history() with 1 min ticker """ @@ -102,30 +102,28 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau _backup_file(file) # do not download a new pair if refresh_pairs isn't set - history.load_pair_history(datadir=None, + history.load_pair_history(datadir=testdatadir, ticker_interval='1m', - refresh_pairs=False, pair='MEME/BTC') assert os.path.isfile(file) is False assert log_has( 'No history data for pair: "MEME/BTC", interval: 1m. ' - 'Use --refresh-pairs-cached option or `freqtrade download-data` ' - 'script to download the data', caplog + 'Use `freqtrade download-data` to download the data', caplog ) # download a new pair if refresh_pairs is set - history.load_pair_history(datadir=None, + history.load_pair_history(datadir=testdatadir, ticker_interval='1m', refresh_pairs=True, exchange=exchange, pair='MEME/BTC') assert os.path.isfile(file) is True - assert log_has( + assert log_has_re( 'Download history data for pair: "MEME/BTC", interval: 1m ' - 'and store in None.', caplog + 'and store in .*', caplog ) with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): - history.load_pair_history(datadir=None, + history.load_pair_history(datadir=testdatadir, ticker_interval='1m', refresh_pairs=True, exchange=None, @@ -133,33 +131,8 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau _clean_test_file(file) -def test_load_data_live(default_conf, mocker, caplog) -> None: - refresh_mock = MagicMock() - mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock) - exchange = get_patched_exchange(mocker, default_conf) - - history.load_data(datadir=None, ticker_interval='5m', - pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'], - live=True, - exchange=exchange) - assert refresh_mock.call_count == 1 - assert len(refresh_mock.call_args_list[0][0][0]) == 2 - assert log_has('Live: Downloading data for all defined pairs ...', caplog) - - -def test_load_data_live_noexchange(default_conf, mocker, caplog) -> None: - - with pytest.raises(OperationalException, - match=r'Exchange needs to be initialized when using live data.'): - history.load_data(datadir=None, ticker_interval='5m', - pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'], - exchange=None, - live=True, - ) - - -def test_testdata_path() -> None: - assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) +def test_testdata_path(testdatadir) -> None: + assert str(Path('tests') / 'testdata') in str(testdatadir) def test_load_cached_data_for_updating(mocker) -> None: @@ -247,7 +220,7 @@ def test_load_cached_data_for_updating(mocker) -> None: assert start_ts is None -def test_download_pair_history(ticker_history_list, mocker, default_conf) -> None: +def test_download_pair_history(ticker_history_list, mocker, default_conf, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') @@ -263,10 +236,10 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf) -> Non assert os.path.isfile(file1_1) is False assert os.path.isfile(file2_1) is False - assert download_pair_history(datadir=None, exchange=exchange, + assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='MEME/BTC', ticker_interval='1m') - assert download_pair_history(datadir=None, exchange=exchange, + assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='CFI/BTC', ticker_interval='1m') assert not exchange._pairs_last_refresh_time @@ -280,10 +253,10 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf) -> Non assert os.path.isfile(file1_5) is False assert os.path.isfile(file2_5) is False - assert download_pair_history(datadir=None, exchange=exchange, + assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='MEME/BTC', ticker_interval='5m') - assert download_pair_history(datadir=None, exchange=exchange, + assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='CFI/BTC', ticker_interval='5m') assert not exchange._pairs_last_refresh_time @@ -295,7 +268,7 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf) -> Non _clean_test_file(file2_5) -def test_download_pair_history2(mocker, default_conf) -> None: +def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: tick = [ [1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839], [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] @@ -303,12 +276,13 @@ def test_download_pair_history2(mocker, default_conf) -> None: json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) - download_pair_history(None, exchange, pair="UNITTEST/BTC", ticker_interval='1m') - download_pair_history(None, exchange, pair="UNITTEST/BTC", ticker_interval='3m') + download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", ticker_interval='1m') + download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", ticker_interval='3m') assert json_dump_mock.call_count == 2 -def test_download_backtesting_data_exception(ticker_history, mocker, caplog, default_conf) -> None: +def test_download_backtesting_data_exception(ticker_history, mocker, caplog, + default_conf, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', side_effect=Exception('File Error')) @@ -319,7 +293,7 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def _backup_file(file1_1) _backup_file(file1_5) - assert not download_pair_history(datadir=None, exchange=exchange, + assert not download_pair_history(datadir=testdatadir, exchange=exchange, pair='MEME/BTC', ticker_interval='1m') # clean files freshly downloaded @@ -331,23 +305,22 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def ) -def test_load_tickerdata_file() -> None: +def test_load_tickerdata_file(testdatadir) -> None: # 7 does not exist in either format. - assert not load_tickerdata_file(None, 'UNITTEST/BTC', '7m') + assert not load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '7m') # 1 exists only as a .json - tickerdata = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') + tickerdata = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m') assert _BTC_UNITTEST_LENGTH == len(tickerdata) # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json - tickerdata = load_tickerdata_file(None, 'UNITTEST/BTC', '8m') + tickerdata = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '8m') assert _BTC_UNITTEST_LENGTH == len(tickerdata) -def test_load_partial_missing(caplog) -> None: +def test_load_partial_missing(testdatadir, caplog) -> None: # Make sure we start fresh - test missing data at start start = arrow.get('2018-01-01T00:00:00') end = arrow.get('2018-01-11T00:00:00') - tickerdata = history.load_data(None, '5m', ['UNITTEST/BTC'], - refresh_pairs=False, + tickerdata = history.load_data(testdatadir, '5m', ['UNITTEST/BTC'], timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) # timedifference in 5 minutes @@ -361,8 +334,8 @@ def test_load_partial_missing(caplog) -> None: caplog.clear() start = arrow.get('2018-01-10T00:00:00') end = arrow.get('2018-02-20T00:00:00') - tickerdata = history.load_data(datadir=None, ticker_interval='5m', - pairs=['UNITTEST/BTC'], refresh_pairs=False, + tickerdata = history.load_data(datadir=testdatadir, ticker_interval='5m', + pairs=['UNITTEST/BTC'], timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) # timedifference in 5 minutes @@ -501,13 +474,13 @@ def test_file_dump_json_tofile() -> None: _clean_test_file(file) -def test_get_timeframe(default_conf, mocker) -> None: +def test_get_timeframe(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) strategy = DefaultStrategy(default_conf) data = strategy.tickerdata_to_dataframe( history.load_data( - datadir=None, + datadir=testdatadir, ticker_interval='1m', pairs=['UNITTEST/BTC'] ) @@ -517,13 +490,13 @@ def test_get_timeframe(default_conf, mocker) -> None: assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' -def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: +def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) -> None: patch_exchange(mocker) strategy = DefaultStrategy(default_conf) data = strategy.tickerdata_to_dataframe( history.load_data( - datadir=None, + datadir=testdatadir, ticker_interval='1m', pairs=['UNITTEST/BTC'], fill_up_missing=False @@ -539,14 +512,14 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: caplog) -def test_validate_backtest_data(default_conf, mocker, caplog) -> None: +def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> None: patch_exchange(mocker) strategy = DefaultStrategy(default_conf) timerange = TimeRange('index', 'index', 200, 250) data = strategy.tickerdata_to_dataframe( history.load_data( - datadir=None, + datadir=testdatadir, ticker_interval='5m', pairs=['UNITTEST/BTC'], timerange=timerange @@ -558,3 +531,43 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', min_date, max_date, timeframe_to_minutes('5m')) assert len(caplog.record_tuples) == 0 + + +def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir): + dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "unlink", MagicMock()) + + ex = get_patched_exchange(mocker, default_conf) + timerange = TimeRange.parse_timerange("20190101-20190102") + refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], + timeframes=["1m", "5m"], dl_path=testdatadir, + timerange=timerange, erase=True + ) + + assert dl_mock.call_count == 4 + assert dl_mock.call_args[1]['timerange'].starttype == 'date' + + assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) + + +def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): + dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + ex = get_patched_exchange(mocker, default_conf) + timerange = TimeRange.parse_timerange("20190101-20190102") + unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], + timeframes=["1m", "5m"], + dl_path=testdatadir, + timerange=timerange, erase=False + ) + + assert dl_mock.call_count == 0 + assert "ETH/BTC" in unav_pairs + assert "XRP/BTC" in unav_pairs + assert log_has("Skipping pair ETH/BTC...", caplog) diff --git a/freqtrade/tests/edge/__init__.py b/tests/edge/__init__.py similarity index 100% rename from freqtrade/tests/edge/__init__.py rename to tests/edge/__init__.py diff --git a/freqtrade/tests/edge/test_edge.py b/tests/edge/test_edge.py similarity index 97% rename from freqtrade/tests/edge/test_edge.py rename to tests/edge/test_edge.py index 09fa1d93e..4fab68591 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -14,10 +14,9 @@ from freqtrade import OperationalException from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.strategy.interface import SellType -from freqtrade.tests.conftest import get_patched_freqtradebot, log_has -from freqtrade.tests.optimize import (BTContainer, BTrade, - _build_backtest_dataframe, - _get_frame_time_from_offset) +from tests.conftest import get_patched_freqtradebot, log_has +from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, + _get_frame_time_from_offset) # Cases to be tested: # 1) Open trade should be removed from the end @@ -156,8 +155,6 @@ def test_edge_results(edge_conf, mocker, caplog, data) -> None: trades = edge._find_trades_for_stoploss_range(frame, 'TEST/BTC', [data.stop_loss]) results = edge._fill_calculable_fields(DataFrame(trades)) if trades else DataFrame() - print(results) - assert len(trades) == len(data.trades) if not results.empty: @@ -291,7 +288,6 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals def test_edge_process_downloaded_data(mocker, edge_conf): - edge_conf['datadir'] = None freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) mocker.patch('freqtrade.data.history.load_data', mocked_load_data) @@ -303,7 +299,6 @@ def test_edge_process_downloaded_data(mocker, edge_conf): def test_edge_process_no_data(mocker, edge_conf, caplog): - edge_conf['datadir'] = None freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) @@ -316,7 +311,6 @@ def test_edge_process_no_data(mocker, edge_conf, caplog): def test_edge_process_no_trades(mocker, edge_conf, caplog): - edge_conf['datadir'] = None freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) mocker.patch('freqtrade.data.history.load_data', mocked_load_data) diff --git a/freqtrade/tests/exchange/__init__.py b/tests/exchange/__init__.py similarity index 100% rename from freqtrade/tests/exchange/__init__.py rename to tests/exchange/__init__.py diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py new file mode 100644 index 000000000..7720a7d2e --- /dev/null +++ b/tests/exchange/test_binance.py @@ -0,0 +1,92 @@ +from random import randint +from unittest.mock import MagicMock + +import ccxt +import pytest + +from freqtrade import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) +from tests.conftest import get_patched_exchange + + +def test_stoploss_limit_order(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'stop_loss_limit' + + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + + default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + + with pytest.raises(OperationalException): + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + + api_mock.create_order.reset_mock() + + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 + assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} + + # test exception handling + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(InvalidOrderException): + api_mock.create_order = MagicMock( + side_effect=ccxt.InvalidOrder("binance Order would trigger immediately.")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(TemporaryError): + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(OperationalException, match=r".*DeadBeef.*"): + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + +def test_stoploss_limit_order_dry_run(default_conf, mocker): + api_mock = MagicMock() + order_type = 'stop_loss_limit' + default_conf['dry_run'] = True + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + + with pytest.raises(OperationalException): + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + + api_mock.create_order.reset_mock() + + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + assert 'id' in order + assert 'info' in order + assert 'type' in order + + assert order['type'] == order_type + assert order['price'] == 220 + assert order['amount'] == 1 diff --git a/freqtrade/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py similarity index 94% rename from freqtrade/tests/exchange/test_exchange.py rename to tests/exchange/test_exchange.py index e453b5dca..f8c556aeb 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -20,7 +20,7 @@ from freqtrade.exchange.exchange import (API_RETRY_COUNT, timeframe_to_minutes, timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.resolvers.exchange_resolver import ExchangeResolver -from freqtrade.tests.conftest import get_patched_exchange, log_has, log_has_re +from tests.conftest import get_patched_exchange, log_has, log_has_re # Make sure to always keep one exchange here which is NOT subclassed!! EXCHANGES = ['bittrex', 'binance', 'kraken', ] @@ -101,18 +101,21 @@ def test_destroy(default_conf, mocker, caplog): def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' - with pytest.raises( - OperationalException, - match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + with pytest.raises(OperationalException, + match=f"Exchange {default_conf['exchange']['name']} is not supported"): Exchange(default_conf) default_conf['exchange']['name'] = 'binance' - with pytest.raises( - OperationalException, - match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + with pytest.raises(OperationalException, + match=f"Exchange {default_conf['exchange']['name']} is not supported"): mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError)) Exchange(default_conf) + with pytest.raises(OperationalException, + match=r"Initialization of ccxt failed. Reason: DeadBeef"): + mocker.patch("ccxt.binance", MagicMock(side_effect=ccxt.BaseError("DeadBeef"))) + Exchange(default_conf) + def test_exchange_resolver(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock())) @@ -864,7 +867,7 @@ def test_get_balance_dry_run(default_conf, mocker): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_get_balance_prod(default_conf, mocker, exchange_name): api_mock = MagicMock() - api_mock.fetch_balance = MagicMock(return_value={'BTC': {'free': 123.4}}) + api_mock.fetch_balance = MagicMock(return_value={'BTC': {'free': 123.4, 'total': 123.4}}) default_conf['dry_run'] = False exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) @@ -880,6 +883,7 @@ def test_get_balance_prod(default_conf, mocker, exchange_name): with pytest.raises(TemporaryError, match=r'.*balance due to malformed exchange response:.*'): exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) + mocker.patch('freqtrade.exchange.Kraken.get_balances', MagicMock(return_value={})) exchange.get_balance(currency='BTC') @@ -1032,7 +1036,6 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls since = 5 * 60 * 500 * 1.8 - print(f"since = {since}") ret = exchange.get_historic_ohlcv(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000)) assert exchange._async_get_candle_history.call_count == 2 @@ -1337,7 +1340,6 @@ def test_get_order(default_conf, mocker, exchange_name): order.myid = 123 exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) exchange._dry_run_open_orders['X'] = order - print(exchange.get_order('X', 'TKN/BTC')) assert exchange.get_order('X', 'TKN/BTC').myid == 123 with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'): @@ -1362,9 +1364,7 @@ def test_get_order(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_name(default_conf, mocker, exchange_name): - mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) - default_conf['exchange']['name'] = exchange_name - exchange = Exchange(default_conf) + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) assert exchange.name == exchange_name.title() assert exchange.id == exchange_name @@ -1436,87 +1436,11 @@ def test_get_fee(default_conf, mocker, exchange_name): 'get_fee', 'calculate_fee') -def test_stoploss_limit_order(default_conf, mocker): - api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - order_type = 'stop_loss_limit' - - api_mock.create_order = MagicMock(return_value={ - 'id': order_id, - 'info': { - 'foo': 'bar' - } - }) - - default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - - with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) - - api_mock.create_order.reset_mock() - - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - assert 'id' in order - assert 'info' in order - assert order['id'] == order_id - assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' - assert api_mock.create_order.call_args[0][1] == order_type - assert api_mock.create_order.call_args[0][2] == 'sell' - assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][4] == 200 - assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} - - # test exception handling - with pytest.raises(DependencyException): - api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) +def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf, 'bittrex') + with pytest.raises(OperationalException, match=r"stoploss_limit is not implemented .*"): exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - with pytest.raises(DependencyException): - api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - with pytest.raises(TemporaryError): - api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - with pytest.raises(OperationalException): - api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - -def test_stoploss_limit_order_dry_run(default_conf, mocker): - api_mock = MagicMock() - order_type = 'stop_loss_limit' - default_conf['dry_run'] = True - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - - with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) - - api_mock.create_order.reset_mock() - - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - assert 'id' in order - assert 'info' in order - assert 'type' in order - - assert order['type'] == order_type - assert order['price'] == 220 - assert order['amount'] == 1 - def test_merge_ft_has_dict(default_conf, mocker): mocker.patch.multiple('freqtrade.exchange.Exchange', @@ -1604,7 +1528,7 @@ def test_timeframe_to_prev_date(): assert timeframe_to_prev_date(interval, date) == result date = datetime.now(tz=timezone.utc) - assert timeframe_to_prev_date("5m", date) < date + assert timeframe_to_prev_date("5m") < date def test_timeframe_to_next_date(): @@ -1629,4 +1553,4 @@ def test_timeframe_to_next_date(): assert timeframe_to_next_date(interval, date) == result date = datetime.now(tz=timezone.utc) - assert timeframe_to_next_date("5m", date) > date + assert timeframe_to_next_date("5m") > date diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py new file mode 100644 index 000000000..3ad62d85a --- /dev/null +++ b/tests/exchange/test_kraken.py @@ -0,0 +1,151 @@ +# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement +# pragma pylint: disable=protected-access +from random import randint +from unittest.mock import MagicMock + +from tests.conftest import get_patched_exchange +from tests.exchange.test_exchange import ccxt_exceptionhandlers + + +def test_buy_kraken_trading_agreement(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'limit' + time_in_force = 'ioc' + api_mock.options = {} + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + default_conf['dry_run'] = False + + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") + + order = exchange.buy(pair='ETH/BTC', ordertype=order_type, + amount=1, rate=200, time_in_force=time_in_force) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'buy' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 + assert api_mock.create_order.call_args[0][5] == {'timeInForce': 'ioc', + 'trading_agreement': 'agree'} + + +def test_sell_kraken_trading_agreement(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) + order_type = 'market' + api_mock.options = {} + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + default_conf['dry_run'] = False + + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") + + order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] is None + assert api_mock.create_order.call_args[0][5] == {'trading_agreement': 'agree'} + + +def test_get_balances_prod(default_conf, mocker): + balance_item = { + 'free': None, + 'total': 10.0, + 'used': 0.0 + } + + api_mock = MagicMock() + api_mock.fetch_balance = MagicMock(return_value={ + '1ST': balance_item.copy(), + '2ST': balance_item.copy(), + '3ST': balance_item.copy(), + '4ST': balance_item.copy(), + }) + kraken_open_orders = [{'symbol': '1ST/EUR', + 'type': 'limit', + 'side': 'sell', + 'price': 20, + 'cost': 0.0, + 'amount': 1.0, + 'filled': 0.0, + 'average': 0.0, + 'remaining': 1.0, + }, + {'status': 'open', + 'symbol': '2ST/EUR', + 'type': 'limit', + 'side': 'sell', + 'price': 20.0, + 'cost': 0.0, + 'amount': 2.0, + 'filled': 0.0, + 'average': 0.0, + 'remaining': 2.0, + }, + {'status': 'open', + 'symbol': '2ST/USD', + 'type': 'limit', + 'side': 'sell', + 'price': 20.0, + 'cost': 0.0, + 'amount': 2.0, + 'filled': 0.0, + 'average': 0.0, + 'remaining': 2.0, + }, + {'status': 'open', + 'symbol': 'BTC/3ST', + 'type': 'limit', + 'side': 'buy', + 'price': 20, + 'cost': 0.0, + 'amount': 3.0, + 'filled': 0.0, + 'average': 0.0, + 'remaining': 3.0, + }] + api_mock.fetch_open_orders = MagicMock(return_value=kraken_open_orders) + default_conf['dry_run'] = False + exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") + balances = exchange.get_balances() + assert len(balances) == 4 + assert balances['1ST']['free'] == 9.0 + assert balances['1ST']['total'] == 10.0 + assert balances['1ST']['used'] == 1.0 + + assert balances['2ST']['free'] == 6.0 + assert balances['2ST']['total'] == 10.0 + assert balances['2ST']['used'] == 4.0 + + assert balances['3ST']['free'] == 7.0 + assert balances['3ST']['total'] == 10.0 + assert balances['3ST']['used'] == 3.0 + + assert balances['4ST']['free'] == 10.0 + assert balances['4ST']['total'] == 10.0 + assert balances['4ST']['used'] == 0.0 + ccxt_exceptionhandlers(mocker, default_conf, api_mock, "kraken", + "get_balances", "fetch_balance") diff --git a/freqtrade/tests/optimize/__init__.py b/tests/optimize/__init__.py similarity index 100% rename from freqtrade/tests/optimize/__init__.py rename to tests/optimize/__init__.py diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py similarity index 96% rename from freqtrade/tests/optimize/test_backtest_detail.py rename to tests/optimize/test_backtest_detail.py index 87f567b4f..d8a4190e2 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -8,11 +8,9 @@ from pandas import DataFrame from freqtrade.data.history import get_timeframe from freqtrade.optimize.backtesting import Backtesting from freqtrade.strategy.interface import SellType -from freqtrade.tests.conftest import patch_exchange -from freqtrade.tests.optimize import (BTContainer, BTrade, - _build_backtest_dataframe, - _get_frame_time_from_offset, - tests_ticker_interval) +from tests.conftest import patch_exchange +from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, + _get_frame_time_from_offset, tests_ticker_interval) # Test 0: Sell with signal sell in candle 3 # Test with Stop-loss at 1% @@ -293,8 +291,8 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: patch_exchange(mocker) frame = _build_backtest_dataframe(data.data) backtesting = Backtesting(default_conf) - backtesting.advise_buy = lambda a, m: frame - backtesting.advise_sell = lambda a, m: frame + backtesting.strategy.advise_buy = lambda a, m: frame + backtesting.strategy.advise_sell = lambda a, m: frame caplog.set_level(logging.DEBUG) pair = "UNITTEST/BTC" @@ -310,7 +308,6 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: 'end_date': max_date, } ) - print(results.T) assert len(results) == len(data.trades) assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py similarity index 91% rename from freqtrade/tests/optimize/test_backtesting.py rename to tests/optimize/test_backtesting.py index 5c942ab72..fa40809d8 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -22,9 +22,8 @@ from freqtrade.optimize.backtesting import Backtesting from freqtrade.state import RunMode from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import SellType -from freqtrade.tests.conftest import (get_args, log_has, log_has_re, - patch_exchange, - patched_configuration_load_config_file) +from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, + patched_configuration_load_config_file) def trim_dictlist(dict_list, num): @@ -34,9 +33,9 @@ def trim_dictlist(dict_list, num): return new -def load_data_test(what): +def load_data_test(what, testdatadir): timerange = TimeRange(None, 'line', 0, -101) - pair = history.load_tickerdata_file(None, ticker_interval='1m', + pair = history.load_tickerdata_file(testdatadir, ticker_interval='1m', pair='UNITTEST/BTC', timerange=timerange) datalen = len(pair) @@ -79,12 +78,12 @@ def load_data_test(what): fill_missing=True)} -def simple_backtest(config, contour, num_results, mocker) -> None: +def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: patch_exchange(mocker) config['ticker_interval'] = '1m' backtesting = Backtesting(config) - data = load_data_test(contour) + data = load_data_test(contour, testdatadir) processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timeframe(processed) assert isinstance(processed, dict) @@ -118,8 +117,8 @@ def _load_pair_as_ticks(pair, tickfreq): # FIX: fixturize this? -def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): - data = history.load_data(datadir=None, ticker_interval='1m', pairs=[pair]) +def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record=None): + data = history.load_data(datadir=datadir, ticker_interval='1m', pairs=[pair]) data = trim_dictlist(data, -201) patch_exchange(mocker) backtesting = Backtesting(conf) @@ -189,16 +188,12 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'position_stacking' not in config assert not log_has('Parameter --enable-position-stacking detected ...', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'export' not in config assert 'runmode' in config assert config['runmode'] == RunMode.BACKTEST -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch( @@ -214,7 +209,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--ticker-interval', '1m', '--enable-position-stacking', '--disable-max-market-positions', - '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo', '--export-filename', 'foo_bar.json' @@ -241,9 +235,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --disable-max-market-positions detected ...', caplog) assert log_has('max_open_trades set to unlimited ...', caplog) - assert 'refresh_pairs' in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) @@ -314,8 +305,8 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None: assert backtesting.config == default_conf assert backtesting.ticker_interval == '5m' assert callable(backtesting.strategy.tickerdata_to_dataframe) - assert callable(backtesting.advise_buy) - assert callable(backtesting.advise_sell) + assert callable(backtesting.strategy.advise_buy) + assert callable(backtesting.strategy.advise_sell) assert isinstance(backtesting.strategy.dp, DataProvider) get_fee.assert_called() assert backtesting.fee == 0.5 @@ -330,7 +321,7 @@ def test_backtesting_init_no_ticker_interval(mocker, default_conf, caplog) -> No patch_exchange(mocker) del default_conf['ticker_interval'] default_conf['strategy_list'] = ['DefaultStrategy', - 'TestStrategy'] + 'SampleStrategy'] mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) with pytest.raises(OperationalException): @@ -339,10 +330,10 @@ def test_backtesting_init_no_ticker_interval(mocker, default_conf, caplog) -> No "or as cli argument `--ticker-interval 5m`", caplog) -def test_tickerdata_to_dataframe_bt(default_conf, mocker) -> None: +def test_tickerdata_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) timerange = TimeRange(None, 'line', 0, -100) - tick = history.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) + tick = history.load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", fill_missing=True)} @@ -452,11 +443,10 @@ def test_generate_text_table_strategyn(default_conf, mocker): '| LTC/BTC | 3 | 30.00 | 90.00 ' '| 1.30000000 | 45.00 | 0:20:00 | 3 | 0 |' ) - print(backtesting._generate_text_table_strategy(all_results=results)) assert backtesting._generate_text_table_strategy(all_results=results) == result_str -def test_backtesting_start(default_conf, mocker, caplog) -> None: +def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) @@ -472,7 +462,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] default_conf['ticker_interval'] = '1m' - default_conf['datadir'] = None + default_conf['datadir'] = testdatadir default_conf['export'] = None default_conf['timerange'] = '-100' @@ -489,7 +479,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: assert log_has(line, caplog) -def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: +def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) @@ -505,7 +495,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] default_conf['ticker_interval'] = "1m" - default_conf['datadir'] = None + default_conf['datadir'] = testdatadir default_conf['export'] = None default_conf['timerange'] = '20180101-20180102' @@ -516,13 +506,13 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: assert log_has('No data found. Terminating.', caplog) -def test_backtest(default_conf, fee, mocker) -> None: +def test_backtest(default_conf, fee, mocker, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) backtesting = Backtesting(default_conf) pair = 'UNITTEST/BTC' timerange = TimeRange(None, 'line', 0, -201) - data = history.load_data(datadir=None, ticker_interval='5m', pairs=['UNITTEST/BTC'], + data = history.load_data(datadir=testdatadir, ticker_interval='5m', pairs=['UNITTEST/BTC'], timerange=timerange) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timeframe(data_processed) @@ -570,14 +560,14 @@ def test_backtest(default_conf, fee, mocker) -> None: t["close_rate"], 6) < round(ln.iloc[0]["high"], 6)) -def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: +def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) backtesting = Backtesting(default_conf) # Run a backtesting for an exiting 1min ticker_interval timerange = TimeRange(None, 'line', 0, -200) - data = history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC'], + data = history.load_data(datadir=testdatadir, ticker_interval='1m', pairs=['UNITTEST/BTC'], timerange=timerange) processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timeframe(processed) @@ -595,21 +585,21 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: assert len(results) == 1 -def test_processed(default_conf, mocker) -> None: +def test_processed(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf) - dict_of_tickerrows = load_data_test('raise') + dict_of_tickerrows = load_data_test('raise', testdatadir) dataframes = backtesting.strategy.tickerdata_to_dataframe(dict_of_tickerrows) dataframe = dataframes['UNITTEST/BTC'] cols = dataframe.columns # assert the dataframe got some of the indicator columns for col in ['close', 'high', 'low', 'open', 'date', - 'ema50', 'ao', 'macd', 'plus_dm']: + 'ema10', 'rsi', 'fastd', 'plus_di']: assert col in cols -def test_backtest_pricecontours(default_conf, fee, mocker) -> None: +def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir) -> None: # TODO: Evaluate usefullness of this, the patterns and buy-signls are unrealistic mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) tests = [['raise', 19], ['lower', 0], ['sine', 35]] @@ -617,49 +607,50 @@ def test_backtest_pricecontours(default_conf, fee, mocker) -> None: default_conf['experimental'] = {"use_sell_signal": True} for [contour, numres] in tests: - simple_backtest(default_conf, contour, numres, mocker) + simple_backtest(default_conf, contour, numres, mocker, testdatadir) -def test_backtest_clash_buy_sell(mocker, default_conf): +def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): # Override the default buy trend function in our default_strategy def fun(dataframe=None, pair=None): buy_value = 1 sell_value = 1 return _trend(dataframe, buy_value, sell_value) - backtest_conf = _make_backtest_conf(mocker, conf=default_conf) + backtest_conf = _make_backtest_conf(mocker, conf=default_conf, datadir=testdatadir) backtesting = Backtesting(default_conf) - backtesting.advise_buy = fun # Override - backtesting.advise_sell = fun # Override + backtesting.strategy.advise_buy = fun # Override + backtesting.strategy.advise_sell = fun # Override results = backtesting.backtest(backtest_conf) assert results.empty -def test_backtest_only_sell(mocker, default_conf): +def test_backtest_only_sell(mocker, default_conf, testdatadir): # Override the default buy trend function in our default_strategy def fun(dataframe=None, pair=None): buy_value = 0 sell_value = 1 return _trend(dataframe, buy_value, sell_value) - backtest_conf = _make_backtest_conf(mocker, conf=default_conf) + backtest_conf = _make_backtest_conf(mocker, conf=default_conf, datadir=testdatadir) backtesting = Backtesting(default_conf) - backtesting.advise_buy = fun # Override - backtesting.advise_sell = fun # Override + backtesting.strategy.advise_buy = fun # Override + backtesting.strategy.advise_sell = fun # Override results = backtesting.backtest(backtest_conf) assert results.empty -def test_backtest_alternate_buy_sell(default_conf, fee, mocker): +def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch('freqtrade.optimize.backtesting.file_dump_json', MagicMock()) - backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC') + backtest_conf = _make_backtest_conf(mocker, conf=default_conf, + pair='UNITTEST/BTC', datadir=testdatadir) # We need to enable sell-signal - otherwise it sells on ROI!! default_conf['experimental'] = {"use_sell_signal": True} default_conf['ticker_interval'] = '1m' backtesting = Backtesting(default_conf) - backtesting.advise_buy = _trend_alternate # Override - backtesting.advise_sell = _trend_alternate # Override + backtesting.strategy.advise_buy = _trend_alternate # Override + backtesting.strategy.advise_sell = _trend_alternate # Override results = backtesting.backtest(backtest_conf) backtesting._store_backtest_result("test_.json", results) # 200 candles in backtest data @@ -672,7 +663,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): @pytest.mark.parametrize("pair", ['ADA/BTC', 'LTC/BTC']) @pytest.mark.parametrize("tres", [0, 20, 30]) -def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair): +def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir): def _trend_alternate_hold(dataframe=None, metadata=None): """ @@ -690,7 +681,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair): patch_exchange(mocker) pairs = ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC', 'NXT/BTC'] - data = history.load_data(datadir=None, ticker_interval='5m', pairs=pairs) + data = history.load_data(datadir=testdatadir, ticker_interval='5m', pairs=pairs) # Only use 500 lines to increase performance data = trim_dictlist(data, -500) @@ -701,8 +692,8 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair): default_conf['ticker_interval'] = '5m' backtesting = Backtesting(default_conf) - backtesting.advise_buy = _trend_alternate_hold # Override - backtesting.advise_sell = _trend_alternate_hold # Override + backtesting.strategy.advise_buy = _trend_alternate_hold # Override + backtesting.strategy.advise_sell = _trend_alternate_hold # Override data_processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timeframe(data_processed) @@ -806,7 +797,7 @@ def test_backtest_record(default_conf, fee, mocker): assert dur > 0 -def test_backtest_start_timerange(default_conf, mocker, caplog): +def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] async def load_pairs(pair, timeframe, since): @@ -823,7 +814,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog): args = [ '--config', 'config.json', '--strategy', 'DefaultStrategy', - '--datadir', 'freqtrade/tests/testdata', + '--datadir', str(testdatadir), 'backtesting', '--ticker-interval', '1m', '--timerange', '-100', @@ -837,7 +828,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog): 'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: -100 ...', - 'Using data directory: freqtrade/tests/testdata ...', + f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' @@ -849,7 +840,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog): assert log_has(line, caplog) -def test_backtest_start_multi_strat(default_conf, mocker, caplog): +def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] async def load_pairs(pair, timeframe, since): @@ -869,7 +860,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): args = [ '--config', 'config.json', - '--datadir', 'freqtrade/tests/testdata', + '--datadir', str(testdatadir), 'backtesting', '--ticker-interval', '1m', '--timerange', '-100', @@ -877,7 +868,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): '--disable-max-market-positions', '--strategy-list', 'DefaultStrategy', - 'TestStrategy', + 'SampleStrategy', ] args = get_args(args) start_backtesting(args) @@ -891,14 +882,14 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): 'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: -100 ...', - 'Using data directory: freqtrade/tests/testdata ...', + f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', - 'Running backtesting for Strategy TestStrategy', + 'Running backtesting for Strategy SampleStrategy', ] for line in exists: diff --git a/freqtrade/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py similarity index 87% rename from freqtrade/tests/optimize/test_edge_cli.py rename to tests/optimize/test_edge_cli.py index 25ad48e43..97103da55 100644 --- a/freqtrade/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -3,15 +3,12 @@ from unittest.mock import MagicMock -import pytest - from freqtrade.edge import PairInfo from freqtrade.optimize import setup_configuration, start_edge from freqtrade.optimize.edge_cli import EdgeCli from freqtrade.state import RunMode -from freqtrade.tests.conftest import (get_args, log_has, log_has_re, - patch_exchange, - patched_configuration_load_config_file) +from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, + patched_configuration_load_config_file) def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: @@ -36,14 +33,10 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'ticker_interval' in config assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'stoploss_range' not in config -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> None: patched_configuration_load_config_file(mocker, edge_conf) mocker.patch( @@ -57,7 +50,6 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N '--datadir', '/foo/bar', 'edge', '--ticker-interval', '1m', - '--refresh-pairs-cached', '--timerange', ':100', '--stoplosses=-0.01,-0.10,-0.001' ] @@ -75,8 +67,6 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', caplog) - assert 'refresh_pairs' in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py similarity index 69% rename from freqtrade/tests/optimize/test_hyperopt.py rename to tests/optimize/test_hyperopt.py index 7b525454c..c9a112422 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,27 +1,26 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import os from datetime import datetime +from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pandas as pd import pytest from arrow import Arrow from filelock import Timeout -from pathlib import Path -from freqtrade import DependencyException, OperationalException +from freqtrade import OperationalException from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.optimize import setup_configuration, start_hyperopt from freqtrade.optimize.default_hyperopt import DefaultHyperOpts from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss from freqtrade.optimize.hyperopt import Hyperopt -from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver, HyperOptLossResolver +from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver, + HyperOptResolver) from freqtrade.state import RunMode from freqtrade.strategy.interface import SellType -from freqtrade.tests.conftest import (get_args, log_has, log_has_re, - patch_exchange, - patched_configuration_load_config_file) +from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, + patched_configuration_load_config_file) @pytest.fixture(scope='function') @@ -36,25 +35,23 @@ def hyperopt_results(): return pd.DataFrame( { 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], - 'profit_percent': [0.1, 0.2, 0.3], - 'profit_abs': [0.2, 0.4, 0.5], + 'profit_percent': [-0.1, 0.2, 0.3], + 'profit_abs': [-0.2, 0.4, 0.6], 'trade_duration': [10, 30, 10], - 'profit': [2, 0, 0], - 'loss': [0, 0, 1], - 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI] } ) # Functions for recurrent object patching -def create_trials(mocker, hyperopt) -> None: +def create_trials(mocker, hyperopt, testdatadir) -> None: """ When creating trials, mock the hyperopt Trials so that *by default* - we don't create any pickle'd files in the filesystem - we might have a pickle'd file so make sure that we return false when looking for it """ - hyperopt.trials_file = Path('freqtrade/tests/optimize/ut_trials.pickle') + hyperopt.trials_file = testdatadir / 'optimize/ut_trials.pickle' mocker.patch.object(Path, "is_file", MagicMock(return_value=False)) stat_mock = MagicMock() @@ -89,15 +86,11 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca assert 'position_stacking' not in config assert not log_has('Parameter --enable-position-stacking detected ...', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'runmode' in config assert config['runmode'] == RunMode.HYPEROPT -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch( @@ -111,7 +104,6 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo 'hyperopt', '--ticker-interval', '1m', '--timerange', ':100', - '--refresh-pairs-cached', '--enable-position-stacking', '--disable-max-market-positions', '--epochs', '1000', @@ -140,9 +132,6 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo assert log_has('Parameter --disable-max-market-positions detected ...', caplog) assert log_has('max_open_trades set to unlimited ...', caplog) - assert 'refresh_pairs' in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) @@ -164,15 +153,15 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None: delattr(hyperopts, 'populate_sell_trend') mocker.patch( 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver._load_hyperopt', - MagicMock(return_value=hyperopts) + MagicMock(return_value=hyperopts(default_conf)) ) x = HyperOptResolver(default_conf, ).hyperopt assert not hasattr(x, 'populate_buy_trend') assert not hasattr(x, 'populate_sell_trend') - assert log_has("Custom Hyperopt does not provide populate_sell_trend. " - "Using populate_sell_trend from DefaultStrategy.", caplog) - assert log_has("Custom Hyperopt does not provide populate_buy_trend. " - "Using populate_buy_trend from DefaultStrategy.", caplog) + assert log_has("Hyperopt class does not provide populate_sell_trend() method. " + "Using populate_sell_trend from the strategy.", caplog) + assert log_has("Hyperopt class does not provide populate_buy_trend() method. " + "Using populate_buy_trend from the strategy.", caplog) assert hasattr(x, "ticker_interval") @@ -201,6 +190,24 @@ def test_hyperoptlossresolver_wrongname(mocker, default_conf, caplog) -> None: HyperOptLossResolver(default_conf, ).hyperopt +def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None: + start_mock = MagicMock() + patched_configuration_load_config_file(mocker, default_conf) + + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) + patch_exchange(mocker) + + args = [ + '--config', 'config.json', + 'hyperopt', + '--epochs', '5' + ] + args = get_args(args) + + with pytest.raises(OperationalException, match=r"Please ensure that the hyperopt dependencies"): + start_hyperopt(args) + + def test_start(mocker, default_conf, caplog) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, default_conf) @@ -215,9 +222,6 @@ def test_start(mocker, default_conf, caplog) -> None: args = get_args(args) start_hyperopt(args) - import pprint - pprint.pprint(caplog.record_tuples) - assert log_has('Starting freqtrade in Hyperopt mode', caplog) assert start_mock.call_count == 1 @@ -240,30 +244,9 @@ def test_start_no_data(mocker, default_conf, caplog) -> None: args = get_args(args) start_hyperopt(args) - import pprint - pprint.pprint(caplog.record_tuples) - assert log_has('No data found. Terminating.', caplog) -def test_start_failure(mocker, default_conf, caplog) -> None: - start_mock = MagicMock() - patched_configuration_load_config_file(mocker, default_conf) - mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) - patch_exchange(mocker) - - args = [ - '--config', 'config.json', - '--strategy', 'TestStrategy', - 'hyperopt', - '--epochs', '5' - ] - args = get_args(args) - with pytest.raises(DependencyException): - start_hyperopt(args) - assert log_has("Please don't use --strategy for hyperopt.", caplog) - - def test_start_filelock(mocker, default_conf, caplog) -> None: start_mock = MagicMock(side_effect=Timeout(Hyperopt.get_lock_filename(default_conf))) patched_configuration_load_config_file(mocker, default_conf) @@ -374,23 +357,23 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: assert caplog.record_tuples == [] -def test_save_trials_saves_trials(mocker, hyperopt, caplog) -> None: - trials = create_trials(mocker, hyperopt) +def test_save_trials_saves_trials(mocker, hyperopt, testdatadir, caplog) -> None: + trials = create_trials(mocker, hyperopt, testdatadir) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) hyperopt.trials = trials hyperopt.save_trials() - trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') - assert log_has('Saving 1 evaluations to \'{}\''.format(trials_file), caplog) + trials_file = testdatadir / 'optimize' / 'ut_trials.pickle' + assert log_has(f"Saving 1 evaluations to '{trials_file}'", caplog) mock_dump.assert_called_once() -def test_read_trials_returns_trials_file(mocker, hyperopt, caplog) -> None: - trials = create_trials(mocker, hyperopt) +def test_read_trials_returns_trials_file(mocker, hyperopt, testdatadir, caplog) -> None: + trials = create_trials(mocker, hyperopt, testdatadir) mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials) hyperopt_trial = hyperopt.read_trials() - trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') - assert log_has('Reading Trials from \'{}\''.format(trials_file), caplog) + trials_file = testdatadir / 'optimize' / 'ut_trials.pickle' + assert log_has(f"Reading Trials from '{trials_file}'", caplog) assert hyperopt_trial == trials mock_load.assert_called_once() @@ -418,7 +401,8 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', + 'params': {'buy': {}, 'sell': {}, 'roi': {}, 'stoploss': 0.0}}]) ) patch_exchange(mocker) @@ -441,8 +425,8 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: assert dumper.called # Should be called twice, once for tickerdata, once to save evaluations assert dumper.call_count == 2 - assert hasattr(hyperopt.backtesting, "advise_sell") - assert hasattr(hyperopt.backtesting, "advise_buy") + assert hasattr(hyperopt.backtesting.strategy, "advise_sell") + assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") assert hyperopt.max_open_trades == default_conf['max_open_trades'] assert hasattr(hyperopt, "position_stacking") @@ -484,8 +468,8 @@ def test_has_space(hyperopt): assert hyperopt.has_space('buy') -def test_populate_indicators(hyperopt) -> None: - tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') +def test_populate_indicators(hyperopt, testdatadir) -> None: + tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", fill_missing=True)} dataframes = hyperopt.backtesting.strategy.tickerdata_to_dataframe(tickerlist) @@ -498,8 +482,8 @@ def test_populate_indicators(hyperopt) -> None: assert 'rsi' in dataframe -def test_buy_strategy_generator(hyperopt) -> None: - tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') +def test_buy_strategy_generator(hyperopt, testdatadir) -> None: + tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", fill_missing=True)} dataframes = hyperopt.backtesting.strategy.tickerdata_to_dataframe(tickerlist) @@ -584,6 +568,7 @@ def test_generate_optimizer(mocker, default_conf) -> None: } hyperopt = Hyperopt(default_conf) + hyperopt.dimensions = hyperopt.hyperopt_space() generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values())) assert generate_optimizer_value == response_expected @@ -693,3 +678,200 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> assert dumper.called # Should be called twice, once for tickerdata, once to save evaluations assert dumper.call_count == 2 + + +def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {'stoploss': 0.0}}]) + ) + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'roi stoploss', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + del hyperopt.custom_hyperopt.__class__.buy_strategy_generator + del hyperopt.custom_hyperopt.__class__.sell_strategy_generator + del hyperopt.custom_hyperopt.__class__.indicator_space + del hyperopt.custom_hyperopt.__class__.sell_indicator_space + + hyperopt.start() + + parallel.assert_called_once() + + out, err = capsys.readouterr() + assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out + assert dumper.called + # Should be called twice, once for tickerdata, once to save evaluations + assert dumper.call_count == 2 + assert hasattr(hyperopt.backtesting.strategy, "advise_sell") + assert hasattr(hyperopt.backtesting.strategy, "advise_buy") + assert hasattr(hyperopt, "max_open_trades") + assert hyperopt.max_open_trades == default_conf['max_open_trades'] + assert hasattr(hyperopt, "position_stacking") + + +def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -> None: + mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'all', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + del hyperopt.custom_hyperopt.__class__.buy_strategy_generator + del hyperopt.custom_hyperopt.__class__.sell_strategy_generator + del hyperopt.custom_hyperopt.__class__.indicator_space + del hyperopt.custom_hyperopt.__class__.sell_indicator_space + + with pytest.raises(OperationalException, match=r"The 'buy' space is included into *"): + hyperopt.start() + + +def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + ) + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'buy', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + # TODO: sell_strategy_generator() is actually not called because + # run_optimizer_parallel() is mocked + del hyperopt.custom_hyperopt.__class__.sell_strategy_generator + del hyperopt.custom_hyperopt.__class__.sell_indicator_space + + hyperopt.start() + + parallel.assert_called_once() + + out, err = capsys.readouterr() + assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out + assert dumper.called + # Should be called twice, once for tickerdata, once to save evaluations + assert dumper.call_count == 2 + assert hasattr(hyperopt.backtesting.strategy, "advise_sell") + assert hasattr(hyperopt.backtesting.strategy, "advise_buy") + assert hasattr(hyperopt, "max_open_trades") + assert hyperopt.max_open_trades == default_conf['max_open_trades'] + assert hasattr(hyperopt, "position_stacking") + + +def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + ) + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'sell', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + # TODO: buy_strategy_generator() is actually not called because + # run_optimizer_parallel() is mocked + del hyperopt.custom_hyperopt.__class__.buy_strategy_generator + del hyperopt.custom_hyperopt.__class__.indicator_space + + hyperopt.start() + + parallel.assert_called_once() + + out, err = capsys.readouterr() + assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out + assert dumper.called + # Should be called twice, once for tickerdata, once to save evaluations + assert dumper.call_count == 2 + assert hasattr(hyperopt.backtesting.strategy, "advise_sell") + assert hasattr(hyperopt.backtesting.strategy, "advise_buy") + assert hasattr(hyperopt, "max_open_trades") + assert hyperopt.max_open_trades == default_conf['max_open_trades'] + assert hasattr(hyperopt, "position_stacking") + + +@pytest.mark.parametrize("method,space", [ + ('buy_strategy_generator', 'buy'), + ('indicator_space', 'buy'), + ('sell_strategy_generator', 'sell'), + ('sell_indicator_space', 'sell'), +]) +def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, method, space) -> None: + mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': space, + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + delattr(hyperopt.custom_hyperopt.__class__, method) + + with pytest.raises(OperationalException, match=f"The '{space}' space is included into *"): + hyperopt.start() diff --git a/freqtrade/tests/pairlist/__init__.py b/tests/pairlist/__init__.py similarity index 100% rename from freqtrade/tests/pairlist/__init__.py rename to tests/pairlist/__init__.py diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py similarity index 99% rename from freqtrade/tests/pairlist/test_pairlist.py rename to tests/pairlist/test_pairlist.py index e7439bb51..411ae60a3 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, PropertyMock from freqtrade import OperationalException from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.resolvers import PairListResolver -from freqtrade.tests.conftest import get_patched_freqtradebot +from tests.conftest import get_patched_freqtradebot import pytest # whitelist, blacklist diff --git a/freqtrade/tests/rpc/__init__.py b/tests/rpc/__init__.py similarity index 100% rename from freqtrade/tests/rpc/__init__.py rename to tests/rpc/__init__.py diff --git a/freqtrade/tests/rpc/test_fiat_convert.py b/tests/rpc/test_fiat_convert.py similarity index 97% rename from freqtrade/tests/rpc/test_fiat_convert.py rename to tests/rpc/test_fiat_convert.py index 1689ecac6..05760ce25 100644 --- a/freqtrade/tests/rpc/test_fiat_convert.py +++ b/tests/rpc/test_fiat_convert.py @@ -8,7 +8,7 @@ import pytest from requests.exceptions import RequestException from freqtrade.rpc.fiat_convert import CryptoFiat, CryptoToFiatConverter -from freqtrade.tests.conftest import log_has +from tests.conftest import log_has def test_pair_convertion_object(): @@ -210,3 +210,10 @@ def test_convert_amount(mocker): fiat_symbol="BTC" ) assert result == 1.23 + + result = fiat_convert.convert_amount( + crypto_amount="1.23", + crypto_symbol="BTC", + fiat_symbol="BTC" + ) + assert result == 1.23 diff --git a/freqtrade/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py similarity index 98% rename from freqtrade/tests/rpc/test_rpc.py rename to tests/rpc/test_rpc.py index 5d3fb7920..66468927f 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -14,7 +14,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State -from freqtrade.tests.conftest import patch_exchange, patch_get_signal +from tests.conftest import patch_exchange, patch_get_signal # Functions for recurrent object patching @@ -28,9 +28,9 @@ def prec_satoshi(a, b) -> float: # Unit tests def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - _load_markets=MagicMock(return_value={}), get_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) @@ -363,9 +363,9 @@ def test_rpc_balance_handle_error(default_conf, mocker): assert 'USD' == result['symbol'] assert result['currencies'] == [{ 'currency': 'BTC', - 'available': 10.0, + 'free': 10.0, 'balance': 12.0, - 'pending': 2.0, + 'used': 2.0, 'est_btc': 12.0, }] assert result['total'] == 12.0 @@ -417,22 +417,22 @@ def test_rpc_balance_handle(default_conf, mocker): assert 'USD' == result['symbol'] assert result['currencies'] == [ {'currency': 'BTC', - 'available': 10.0, + 'free': 10.0, 'balance': 12.0, - 'pending': 2.0, + 'used': 2.0, 'est_btc': 12.0, }, - {'available': 1.0, + {'free': 1.0, 'balance': 5.0, 'currency': 'ETH', 'est_btc': 0.05, - 'pending': 4.0 + 'used': 4.0 }, - {'available': 5.0, + {'free': 5.0, 'balance': 10.0, 'currency': 'PAX', 'est_btc': 0.1, - 'pending': 5.0} + 'used': 5.0} ] assert result['total'] == 12.15 diff --git a/freqtrade/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py similarity index 99% rename from freqtrade/tests/rpc/test_rpc_apiserver.py rename to tests/rpc/test_rpc_apiserver.py index 794343a98..b572a0514 100644 --- a/freqtrade/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -13,9 +13,7 @@ from freqtrade.__init__ import __version__ from freqtrade.persistence import Trade from freqtrade.rpc.api_server import BASE_URI, ApiServer from freqtrade.state import State -from freqtrade.tests.conftest import (get_patched_freqtradebot, log_has, - patch_get_signal) - +from tests.conftest import get_patched_freqtradebot, log_has, patch_get_signal _TEST_USER = "FreqTrader" _TEST_PASS = "SuperSecurePassword1!" @@ -251,9 +249,9 @@ def test_api_balance(botclient, mocker, rpc_balance): assert len(rc.json["currencies"]) == 5 assert rc.json['currencies'][0] == { 'currency': 'BTC', - 'available': 12.0, + 'free': 12.0, 'balance': 12.0, - 'pending': 0.0, + 'used': 0.0, 'est_btc': 12.0, } diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py similarity index 88% rename from freqtrade/tests/rpc/test_rpc_manager.py rename to tests/rpc/test_rpc_manager.py index 468e3e8e5..7278f0671 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -4,7 +4,7 @@ import logging from unittest.mock import MagicMock from freqtrade.rpc import RPCMessageType, RPCManager -from freqtrade.tests.conftest import log_has, get_patched_freqtradebot +from tests.conftest import log_has, get_patched_freqtradebot def test__init__(mocker, default_conf) -> None: @@ -115,6 +115,22 @@ def test_init_webhook_enabled(mocker, default_conf, caplog) -> None: assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] +def test_send_msg_webhook_CustomMessagetype(mocker, default_conf, caplog) -> None: + caplog.set_level(logging.DEBUG) + default_conf['telegram']['enabled'] = False + default_conf['webhook'] = {'enabled': True, 'url': "https://DEADBEEF.com"} + mocker.patch('freqtrade.rpc.webhook.Webhook.send_msg', + MagicMock(side_effect=NotImplementedError)) + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) + + assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] + rpc_manager.send_msg({'type': RPCMessageType.CUSTOM_NOTIFICATION, + 'status': 'TestMessage'}) + assert log_has( + "Message type RPCMessageType.CUSTOM_NOTIFICATION not implemented by handler webhook.", + caplog) + + def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py similarity index 91% rename from freqtrade/tests/rpc/test_rpc_telegram.py rename to tests/rpc/test_rpc_telegram.py index 2f469643f..a776ad5df 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -21,8 +21,8 @@ from freqtrade.rpc import RPCMessageType from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import State from freqtrade.strategy.interface import SellType -from freqtrade.tests.conftest import (get_patched_freqtradebot, log_has, - patch_exchange, patch_get_signal) +from tests.conftest import (get_patched_freqtradebot, log_has, patch_exchange, + patch_get_signal) class DummyCls(Telegram): @@ -90,7 +90,7 @@ def test_cleanup(default_conf, mocker) -> None: def test_authorized_only(default_conf, mocker, caplog) -> None: - patch_exchange(mocker, None) + patch_exchange(mocker) chat = Chat(0, 0) update = Update(randint(1, 100)) @@ -100,7 +100,7 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: bot = FreqtradeBot(default_conf) patch_get_signal(bot, (True, False)) dummy = DummyCls(bot) - dummy.dummy_handler(bot=MagicMock(), update=update) + dummy.dummy_handler(update=update, context=MagicMock()) assert dummy.state['called'] is True assert log_has('Executing handler: dummy_handler for chat_id: 0', caplog) assert not log_has('Rejected unauthorized message from: 0', caplog) @@ -108,7 +108,7 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: - patch_exchange(mocker, None) + patch_exchange(mocker) chat = Chat(0xdeadbeef, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) @@ -117,7 +117,7 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: bot = FreqtradeBot(default_conf) patch_get_signal(bot, (True, False)) dummy = DummyCls(bot) - dummy.dummy_handler(bot=MagicMock(), update=update) + dummy.dummy_handler(update=update, context=MagicMock()) assert dummy.state['called'] is False assert not log_has('Executing handler: dummy_handler for chat_id: 3735928559', caplog) assert log_has('Rejected unauthorized message from: 3735928559', caplog) @@ -136,7 +136,7 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: patch_get_signal(bot, (True, False)) dummy = DummyCls(bot) - dummy.dummy_exception(bot=MagicMock(), update=update) + dummy.dummy_exception(update=update, context=MagicMock()) assert dummy.state['called'] is False assert not log_has('Executing handler: dummy_handler for chat_id: 0', caplog) assert not log_has('Rejected unauthorized message from: 0', caplog) @@ -194,12 +194,13 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: for _ in range(3): freqtradebot.create_trades() - telegram._status(bot=MagicMock(), update=update) + telegram._status(update=update, context=MagicMock()) assert msg_mock.call_count == 1 - update.message.text = MagicMock() - update.message.text.replace = MagicMock(return_value='table 2 3') - telegram._status(bot=MagicMock(), update=update) + context = MagicMock() + # /status table 2 3 + context.args = ["table", "2", "3"] + telegram._status(update=update, context=context) assert status_table.call_count == 1 @@ -228,13 +229,13 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No freqtradebot.state = State.STOPPED # Status is also enabled when stopped - telegram._status(bot=MagicMock(), update=update) + telegram._status(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'no active trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() freqtradebot.state = State.RUNNING - telegram._status(bot=MagicMock(), update=update) + telegram._status(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'no active trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() @@ -242,7 +243,7 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No # Create some test data freqtradebot.create_trades() # Trigger status while we have a fulfilled order for the open trade - telegram._status(bot=MagicMock(), update=update) + telegram._status(update=update, context=MagicMock()) # close_rate should not be included in the message as the trade is not closed # and no line should be empty @@ -280,13 +281,13 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) freqtradebot.state = State.STOPPED # Status table is also enabled when stopped - telegram._status_table(bot=MagicMock(), update=update) + telegram._status_table(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'no active order' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() freqtradebot.state = State.RUNNING - telegram._status_table(bot=MagicMock(), update=update) + telegram._status_table(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'no active order' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() @@ -294,7 +295,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) # Create some test data freqtradebot.create_trades() - telegram._status_table(bot=MagicMock(), update=update) + telegram._status_table(update=update, context=MagicMock()) text = re.sub('', '', msg_mock.call_args_list[-1][0][0]) line = text.split("\n") @@ -346,8 +347,10 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, trade.is_open = False # Try valid data - update.message.text = '/daily 2' - telegram._daily(bot=MagicMock(), update=update) + # /daily 2 + context = MagicMock() + context.args = ["2"] + telegram._daily(update=update, context=context) assert msg_mock.call_count == 1 assert 'Daily' in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] @@ -369,9 +372,10 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, trade.close_date = datetime.utcnow() trade.is_open = False - update.message.text = '/daily 1' - - telegram._daily(bot=MagicMock(), update=update) + # /daily 1 + context = MagicMock() + context.args = ["1"] + telegram._daily(update=update, context=context) assert str(' 0.00018651 BTC') in msg_mock.call_args_list[0][0][0] assert str(' 2.798 USD') in msg_mock.call_args_list[0][0][0] assert str(' 3 trades') in msg_mock.call_args_list[0][0][0] @@ -398,16 +402,20 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: # Try invalid data msg_mock.reset_mock() freqtradebot.state = State.RUNNING - update.message.text = '/daily -2' - telegram._daily(bot=MagicMock(), update=update) + # /daily -2 + context = MagicMock() + context.args = ["-2"] + telegram._daily(update=update, context=context) assert msg_mock.call_count == 1 assert 'must be an integer greater than 0' in msg_mock.call_args_list[0][0][0] # Try invalid data msg_mock.reset_mock() freqtradebot.state = State.RUNNING - update.message.text = '/daily today' - telegram._daily(bot=MagicMock(), update=update) + # /daily today + context = MagicMock() + context.args = ["today"] + telegram._daily(update=update, context=context) assert str('Daily Profit over the last 7 days') in msg_mock.call_args_list[0][0][0] @@ -433,7 +441,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) - telegram._profit(bot=MagicMock(), update=update) + telegram._profit(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'no closed trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() @@ -445,7 +453,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) - telegram._profit(bot=MagicMock(), update=update) + telegram._profit(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'no closed trade' in msg_mock.call_args_list[-1][0][0] msg_mock.reset_mock() @@ -457,7 +465,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, trade.close_date = datetime.utcnow() trade.is_open = False - telegram._profit(bot=MagicMock(), update=update) + telegram._profit(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert '*ROI:* Close trades' in msg_mock.call_args_list[-1][0][0] assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] @@ -507,7 +515,7 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance) -> N telegram = Telegram(freqtradebot) - telegram._balance(bot=MagicMock(), update=update) + telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 assert '*BTC:*' in result @@ -536,7 +544,7 @@ def test_balance_handle_empty_response(default_conf, update, mocker) -> None: telegram = Telegram(freqtradebot) freqtradebot.config['dry_run'] = False - telegram._balance(bot=MagicMock(), update=update) + telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 assert 'All balances are zero.' in result @@ -557,7 +565,7 @@ def test_balance_handle_empty_response_dry(default_conf, update, mocker) -> None telegram = Telegram(freqtradebot) - telegram._balance(bot=MagicMock(), update=update) + telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 assert "Running in Dry Run, balances are not available." in result @@ -569,8 +577,8 @@ def test_balance_handle_too_large_response(default_conf, update, mocker) -> None curr = choice(ascii_uppercase) + choice(ascii_uppercase) + choice(ascii_uppercase) balances.append({ 'currency': curr, - 'available': 1.0, - 'pending': 0.5, + 'free': 1.0, + 'used': 0.5, 'balance': i, 'est_btc': 1 }) @@ -593,7 +601,7 @@ def test_balance_handle_too_large_response(default_conf, update, mocker) -> None telegram = Telegram(freqtradebot) - telegram._balance(bot=MagicMock(), update=update) + telegram._balance(update=update, context=MagicMock()) assert msg_mock.call_count > 1 # Test if wrap happens around 4000 - # and each single currency-output is around 120 characters long so we need @@ -615,7 +623,7 @@ def test_start_handle(default_conf, update, mocker) -> None: freqtradebot.state = State.STOPPED assert freqtradebot.state == State.STOPPED - telegram._start(bot=MagicMock(), update=update) + telegram._start(update=update, context=MagicMock()) assert freqtradebot.state == State.RUNNING assert msg_mock.call_count == 1 @@ -633,7 +641,7 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: freqtradebot.state = State.RUNNING assert freqtradebot.state == State.RUNNING - telegram._start(bot=MagicMock(), update=update) + telegram._start(update=update, context=MagicMock()) assert freqtradebot.state == State.RUNNING assert msg_mock.call_count == 1 assert 'already running' in msg_mock.call_args_list[0][0][0] @@ -652,7 +660,7 @@ def test_stop_handle(default_conf, update, mocker) -> None: freqtradebot.state = State.RUNNING assert freqtradebot.state == State.RUNNING - telegram._stop(bot=MagicMock(), update=update) + telegram._stop(update=update, context=MagicMock()) assert freqtradebot.state == State.STOPPED assert msg_mock.call_count == 1 assert 'stopping trader' in msg_mock.call_args_list[0][0][0] @@ -671,7 +679,7 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: freqtradebot.state = State.STOPPED assert freqtradebot.state == State.STOPPED - telegram._stop(bot=MagicMock(), update=update) + telegram._stop(update=update, context=MagicMock()) assert freqtradebot.state == State.STOPPED assert msg_mock.call_count == 1 assert 'already stopped' in msg_mock.call_args_list[0][0][0] @@ -689,7 +697,7 @@ def test_stopbuy_handle(default_conf, update, mocker) -> None: telegram = Telegram(freqtradebot) assert freqtradebot.config['max_open_trades'] != 0 - telegram._stopbuy(bot=MagicMock(), update=update) + telegram._stopbuy(update=update, context=MagicMock()) assert freqtradebot.config['max_open_trades'] == 0 assert msg_mock.call_count == 1 assert 'No more buy will occur from now. Run /reload_conf to reset.' \ @@ -709,7 +717,7 @@ def test_reload_conf_handle(default_conf, update, mocker) -> None: freqtradebot.state = State.RUNNING assert freqtradebot.state == State.RUNNING - telegram._reload_conf(bot=MagicMock(), update=update) + telegram._reload_conf(update=update, context=MagicMock()) assert freqtradebot.state == State.RELOAD_CONF assert msg_mock.call_count == 1 assert 'reloading config' in msg_mock.call_args_list[0][0][0] @@ -720,13 +728,12 @@ def test_forcesell_handle(default_conf, update, ticker, fee, mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - _load_markets=MagicMock(return_value={}), get_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets), - validate_pairs=MagicMock(return_value={}) ) freqtradebot = FreqtradeBot(default_conf) @@ -742,8 +749,10 @@ def test_forcesell_handle(default_conf, update, ticker, fee, # Increase the price and sell it mocker.patch('freqtrade.exchange.Exchange.get_ticker', ticker_sell_up) - update.message.text = '/forcesell 1' - telegram._forcesell(bot=MagicMock(), update=update) + # /forcesell 1 + context = MagicMock() + context.args = ["1"] + telegram._forcesell(update=update, context=context) assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] @@ -771,13 +780,12 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - _load_markets=MagicMock(return_value={}), get_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets), - validate_pairs=MagicMock(return_value={}) ) freqtradebot = FreqtradeBot(default_conf) @@ -796,8 +804,10 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, trade = Trade.query.first() assert trade - update.message.text = '/forcesell 1' - telegram._forcesell(bot=MagicMock(), update=update) + # /forcesell 1 + context = MagicMock() + context.args = ["1"] + telegram._forcesell(update=update, context=context) assert rpc_mock.call_count == 2 @@ -831,7 +841,6 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker get_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets), - validate_pairs=MagicMock(return_value={}) ) default_conf['max_open_trades'] = 4 freqtradebot = FreqtradeBot(default_conf) @@ -842,8 +851,10 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker freqtradebot.create_trades() rpc_mock.reset_mock() - update.message.text = '/forcesell all' - telegram._forcesell(bot=MagicMock(), update=update) + # /forcesell all + context = MagicMock() + context.args = ["all"] + telegram._forcesell(update=update, context=context) assert rpc_mock.call_count == 4 msg = rpc_mock.call_args_list[0][0][0] @@ -882,24 +893,29 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: # Trader is not running freqtradebot.state = State.STOPPED - update.message.text = '/forcesell 1' - telegram._forcesell(bot=MagicMock(), update=update) + # /forcesell 1 + context = MagicMock() + context.args = ["1"] + telegram._forcesell(update=update, context=context) assert msg_mock.call_count == 1 assert 'not running' in msg_mock.call_args_list[0][0][0] # No argument msg_mock.reset_mock() freqtradebot.state = State.RUNNING - update.message.text = '/forcesell' - telegram._forcesell(bot=MagicMock(), update=update) + context = MagicMock() + context.args = [] + telegram._forcesell(update=update, context=context) assert msg_mock.call_count == 1 assert 'invalid argument' in msg_mock.call_args_list[0][0][0] # Invalid argument msg_mock.reset_mock() freqtradebot.state = State.RUNNING - update.message.text = '/forcesell 123456' - telegram._forcesell(bot=MagicMock(), update=update) + # /forcesell 123456 + context = MagicMock() + context.args = ["123456"] + telegram._forcesell(update=update, context=context) assert msg_mock.call_count == 1 assert 'invalid argument' in msg_mock.call_args_list[0][0][0] @@ -908,11 +924,10 @@ def test_forcebuy_handle(default_conf, update, markets, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram._send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - _load_markets=MagicMock(return_value={}), markets=PropertyMock(markets), - validate_pairs=MagicMock(return_value={}) ) fbuy_mock = MagicMock(return_value=None) mocker.patch('freqtrade.rpc.RPC._rpc_forcebuy', fbuy_mock) @@ -921,8 +936,10 @@ def test_forcebuy_handle(default_conf, update, markets, mocker) -> None: patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) - update.message.text = '/forcebuy ETH/BTC' - telegram._forcebuy(bot=MagicMock(), update=update) + # /forcebuy ETH/BTC + context = MagicMock() + context.args = ["ETH/BTC"] + telegram._forcebuy(update=update, context=context) assert fbuy_mock.call_count == 1 assert fbuy_mock.call_args_list[0][0][0] == 'ETH/BTC' @@ -931,8 +948,10 @@ def test_forcebuy_handle(default_conf, update, markets, mocker) -> None: # Reset and retry with specified price fbuy_mock = MagicMock(return_value=None) mocker.patch('freqtrade.rpc.RPC._rpc_forcebuy', fbuy_mock) - update.message.text = '/forcebuy ETH/BTC 0.055' - telegram._forcebuy(bot=MagicMock(), update=update) + # /forcebuy ETH/BTC 0.055 + context = MagicMock() + context.args = ["ETH/BTC", "0.055"] + telegram._forcebuy(update=update, context=context) assert fbuy_mock.call_count == 1 assert fbuy_mock.call_args_list[0][0][0] == 'ETH/BTC' @@ -944,18 +963,17 @@ def test_forcebuy_handle_exception(default_conf, update, markets, mocker) -> Non mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram._send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - _load_markets=MagicMock(return_value={}), markets=PropertyMock(markets), - validate_pairs=MagicMock(return_value={}) ) freqtradebot = FreqtradeBot(default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) update.message.text = '/forcebuy ETH/Nonepair' - telegram._forcebuy(bot=MagicMock(), update=update) + telegram._forcebuy(update=update, context=MagicMock()) assert rpc_mock.call_count == 1 assert rpc_mock.call_args_list[0][0][0] == 'Forcebuy not enabled.' @@ -975,7 +993,6 @@ def test_performance_handle(default_conf, update, ticker, fee, get_ticker=ticker, get_fee=fee, markets=PropertyMock(markets), - validate_pairs=MagicMock(return_value={}) ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) @@ -995,7 +1012,7 @@ def test_performance_handle(default_conf, update, ticker, fee, trade.close_date = datetime.utcnow() trade.is_open = False - telegram._performance(bot=MagicMock(), update=update) + telegram._performance(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'Performance' in msg_mock.call_args_list[0][0][0] assert 'ETH/BTC\t6.20% (1)' in msg_mock.call_args_list[0][0][0] @@ -1021,7 +1038,7 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED - telegram._count(bot=MagicMock(), update=update) + telegram._count(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'not running' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() @@ -1030,7 +1047,7 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non # Create some test data freqtradebot.create_trades() msg_mock.reset_mock() - telegram._count(bot=MagicMock(), update=update) + telegram._count(update=update, context=MagicMock()) msg = '
  current    max    total stake\n---------  -----  -------------\n' \
           '        1      {}          {}
'\ @@ -1052,7 +1069,7 @@ def test_whitelist_static(default_conf, update, mocker) -> None: telegram = Telegram(freqtradebot) - telegram._whitelist(bot=MagicMock(), update=update) + telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert ('Using whitelist `StaticPairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' in msg_mock.call_args_list[0][0][0]) @@ -1073,7 +1090,7 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: telegram = Telegram(freqtradebot) - telegram._whitelist(bot=MagicMock(), update=update) + telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert ('Using whitelist `VolumePairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' in msg_mock.call_args_list[0][0][0]) @@ -1090,13 +1107,17 @@ def test_blacklist_static(default_conf, update, mocker) -> None: telegram = Telegram(freqtradebot) - telegram._blacklist(bot=MagicMock(), update=update, args=[]) + telegram._blacklist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert ("Blacklist contains 2 pairs\n`DOGE/BTC, HOT/BTC`" in msg_mock.call_args_list[0][0][0]) msg_mock.reset_mock() - telegram._blacklist(bot=MagicMock(), update=update, args=["ETH/BTC"]) + + # /blacklist ETH/BTC + context = MagicMock() + context.args = ["ETH/BTC"] + telegram._blacklist(update=update, context=context) assert msg_mock.call_count == 1 assert ("Blacklist contains 3 pairs\n`DOGE/BTC, HOT/BTC, ETH/BTC`" in msg_mock.call_args_list[0][0][0]) @@ -1115,7 +1136,7 @@ def test_edge_disabled(default_conf, update, mocker) -> None: telegram = Telegram(freqtradebot) - telegram._edge(bot=MagicMock(), update=update) + telegram._edge(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert "Edge is not enabled." in msg_mock.call_args_list[0][0][0] @@ -1137,7 +1158,7 @@ def test_edge_enabled(edge_conf, update, mocker) -> None: telegram = Telegram(freqtradebot) - telegram._edge(bot=MagicMock(), update=update) + telegram._edge(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'Edge only validated following pairs:\n
' in msg_mock.call_args_list[0][0][0]
     assert 'Pair      Winrate    Expectancy    Stoploss' in msg_mock.call_args_list[0][0][0]
@@ -1154,7 +1175,7 @@ def test_help_handle(default_conf, update, mocker) -> None:
 
     telegram = Telegram(freqtradebot)
 
-    telegram._help(bot=MagicMock(), update=update)
+    telegram._help(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
     assert '*/help:* `This help message`' in msg_mock.call_args_list[0][0][0]
 
@@ -1169,7 +1190,7 @@ def test_version_handle(default_conf, update, mocker) -> None:
     freqtradebot = get_patched_freqtradebot(mocker, default_conf)
     telegram = Telegram(freqtradebot)
 
-    telegram._version(bot=MagicMock(), update=update)
+    telegram._version(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
     assert '*Version:* `{}`'.format(__version__) in msg_mock.call_args_list[0][0][0]
 
@@ -1395,9 +1416,11 @@ def test__send_msg(default_conf, mocker) -> None:
     bot = MagicMock()
     freqtradebot = get_patched_freqtradebot(mocker, default_conf)
     telegram = Telegram(freqtradebot)
+    telegram._updater = MagicMock()
+    telegram._updater.bot = bot
 
     telegram._config['telegram']['enabled'] = True
-    telegram._send_msg('test', bot)
+    telegram._send_msg('test')
     assert len(bot.method_calls) == 1
 
 
@@ -1407,9 +1430,11 @@ def test__send_msg_network_error(default_conf, mocker, caplog) -> None:
     bot.send_message = MagicMock(side_effect=NetworkError('Oh snap'))
     freqtradebot = get_patched_freqtradebot(mocker, default_conf)
     telegram = Telegram(freqtradebot)
+    telegram._updater = MagicMock()
+    telegram._updater.bot = bot
 
     telegram._config['telegram']['enabled'] = True
-    telegram._send_msg('test', bot)
+    telegram._send_msg('test')
 
     # Bot should've tried to send it twice
     assert len(bot.method_calls) == 2
diff --git a/freqtrade/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py
similarity index 84%
rename from freqtrade/tests/rpc/test_rpc_webhook.py
rename to tests/rpc/test_rpc_webhook.py
index cc491d4dd..dbbc4cefb 100644
--- a/freqtrade/tests/rpc/test_rpc_webhook.py
+++ b/tests/rpc/test_rpc_webhook.py
@@ -8,7 +8,7 @@ from requests import RequestException
 from freqtrade.rpc import RPCMessageType
 from freqtrade.rpc.webhook import Webhook
 from freqtrade.strategy.interface import SellType
-from freqtrade.tests.conftest import get_patched_freqtradebot, log_has
+from tests.conftest import get_patched_freqtradebot, log_has
 
 
 def get_webhook_dict() -> dict:
@@ -91,21 +91,24 @@ def test_send_msg(default_conf, mocker):
     assert (msg_mock.call_args[0][0]["value3"] ==
             default_conf["webhook"]["webhooksell"]["value3"].format(**msg))
 
-    # Test notification
-    msg = {
-        'type': RPCMessageType.STATUS_NOTIFICATION,
-        'status': 'Unfilled sell order for BTC cancelled due to timeout'
-    }
-    msg_mock = MagicMock()
-    mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
-    webhook.send_msg(msg)
-    assert msg_mock.call_count == 1
-    assert (msg_mock.call_args[0][0]["value1"] ==
-            default_conf["webhook"]["webhookstatus"]["value1"].format(**msg))
-    assert (msg_mock.call_args[0][0]["value2"] ==
-            default_conf["webhook"]["webhookstatus"]["value2"].format(**msg))
-    assert (msg_mock.call_args[0][0]["value3"] ==
-            default_conf["webhook"]["webhookstatus"]["value3"].format(**msg))
+    for msgtype in [RPCMessageType.STATUS_NOTIFICATION,
+                    RPCMessageType.WARNING_NOTIFICATION,
+                    RPCMessageType.CUSTOM_NOTIFICATION]:
+        # Test notification
+        msg = {
+            'type': msgtype,
+            'status': 'Unfilled sell order for BTC cancelled due to timeout'
+        }
+        msg_mock = MagicMock()
+        mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
+        webhook.send_msg(msg)
+        assert msg_mock.call_count == 1
+        assert (msg_mock.call_args[0][0]["value1"] ==
+                default_conf["webhook"]["webhookstatus"]["value1"].format(**msg))
+        assert (msg_mock.call_args[0][0]["value2"] ==
+                default_conf["webhook"]["webhookstatus"]["value2"].format(**msg))
+        assert (msg_mock.call_args[0][0]["value3"] ==
+                default_conf["webhook"]["webhookstatus"]["value3"].format(**msg))
 
 
 def test_exception_send_msg(default_conf, mocker, caplog):
diff --git a/freqtrade/tests/strategy/__init__.py b/tests/strategy/__init__.py
similarity index 100%
rename from freqtrade/tests/strategy/__init__.py
rename to tests/strategy/__init__.py
diff --git a/freqtrade/tests/strategy/legacy_strategy.py b/tests/strategy/legacy_strategy.py
similarity index 99%
rename from freqtrade/tests/strategy/legacy_strategy.py
rename to tests/strategy/legacy_strategy.py
index 2cd13b791..af1b617a6 100644
--- a/freqtrade/tests/strategy/legacy_strategy.py
+++ b/tests/strategy/legacy_strategy.py
@@ -15,7 +15,7 @@ class TestStrategyLegacy(IStrategy):
     """
     This is a test strategy using the legacy function headers, which will be
     removed in a future update.
-    Please do not use this as a template, but refer to user_data/strategy/TestStrategy.py
+    Please do not use this as a template, but refer to user_data/strategy/sample_strategy.py
     for a uptodate version of this template.
 
     """
diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py
similarity index 74%
rename from freqtrade/tests/strategy/test_default_strategy.py
rename to tests/strategy/test_default_strategy.py
index 74c81882a..17d6b8ee0 100644
--- a/freqtrade/tests/strategy/test_default_strategy.py
+++ b/tests/strategy/test_default_strategy.py
@@ -1,19 +1,8 @@
-import json
-
-import pytest
 from pandas import DataFrame
 
-from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.strategy.default_strategy import DefaultStrategy
 
 
-@pytest.fixture
-def result():
-    with open('freqtrade/tests/testdata/ETH_BTC-1m.json') as data_file:
-        return parse_ticker_dataframe(json.load(data_file), '1m', pair="UNITTEST/BTC",
-                                      fill_missing=True)
-
-
 def test_default_strategy_structure():
     assert hasattr(DefaultStrategy, 'minimal_roi')
     assert hasattr(DefaultStrategy, 'stoploss')
diff --git a/freqtrade/tests/strategy/test_interface.py b/tests/strategy/test_interface.py
similarity index 98%
rename from freqtrade/tests/strategy/test_interface.py
rename to tests/strategy/test_interface.py
index 36c9ffcd4..094cd41a1 100644
--- a/freqtrade/tests/strategy/test_interface.py
+++ b/tests/strategy/test_interface.py
@@ -10,7 +10,7 @@ from freqtrade.configuration import TimeRange
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.data.history import load_tickerdata_file
 from freqtrade.persistence import Trade
-from freqtrade.tests.conftest import get_patched_exchange, log_has
+from tests.conftest import get_patched_exchange, log_has
 from freqtrade.strategy.default_strategy import DefaultStrategy
 
 # Avoid to reinit the same object again and again
@@ -103,11 +103,11 @@ def test_get_signal_handles_exceptions(mocker, default_conf):
     assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, False)
 
 
-def test_tickerdata_to_dataframe(default_conf) -> None:
+def test_tickerdata_to_dataframe(default_conf, testdatadir) -> None:
     strategy = DefaultStrategy(default_conf)
 
     timerange = TimeRange(None, 'line', 0, -100)
-    tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange)
+    tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange)
     tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC",
                                                          fill_missing=True)}
     data = strategy.tickerdata_to_dataframe(tickerlist)
diff --git a/freqtrade/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py
similarity index 86%
rename from freqtrade/tests/strategy/test_strategy.py
rename to tests/strategy/test_strategy.py
index 240b83b8b..6992d1aa5 100644
--- a/freqtrade/tests/strategy/test_strategy.py
+++ b/tests/strategy/test_strategy.py
@@ -5,43 +5,19 @@ import warnings
 from base64 import urlsafe_b64encode
 from os import path
 from pathlib import Path
-from unittest.mock import Mock
 
 import pytest
 from pandas import DataFrame
 
 from freqtrade import OperationalException
 from freqtrade.resolvers import StrategyResolver
-from freqtrade.strategy import import_strategy
-from freqtrade.strategy.default_strategy import DefaultStrategy
 from freqtrade.strategy.interface import IStrategy
-from freqtrade.tests.conftest import log_has, log_has_re
-
-
-def test_import_strategy(caplog):
-    caplog.set_level(logging.DEBUG)
-    default_config = {}
-
-    strategy = DefaultStrategy(default_config)
-    strategy.some_method = lambda *args, **kwargs: 42
-
-    assert strategy.__module__ == 'freqtrade.strategy.default_strategy'
-    assert strategy.some_method() == 42
-
-    imported_strategy = import_strategy(strategy, default_config)
-
-    assert dir(strategy) == dir(imported_strategy)
-
-    assert imported_strategy.__module__ == 'freqtrade.strategy'
-    assert imported_strategy.some_method() == 42
-
-    assert log_has('Imported strategy freqtrade.strategy.default_strategy.DefaultStrategy '
-                   'as freqtrade.strategy.DefaultStrategy', caplog)
+from tests.conftest import log_has, log_has_re
 
 
 def test_search_strategy():
     default_config = {}
-    default_location = Path(__file__).parent.parent.parent.joinpath('strategy').resolve()
+    default_location = Path(__file__).parent.parent.joinpath('strategy').resolve()
 
     s, _ = StrategyResolver._search_object(
         directory=default_location,
@@ -61,27 +37,27 @@ def test_search_strategy():
 
 
 def test_load_strategy(default_conf, result):
-    default_conf.update({'strategy': 'TestStrategy'})
+    default_conf.update({'strategy': 'SampleStrategy'})
     resolver = StrategyResolver(default_conf)
     assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'})
 
 
 def test_load_strategy_base64(result, caplog, default_conf):
-    with open("user_data/strategies/test_strategy.py", "rb") as file:
+    with open("user_data/strategies/sample_strategy.py", "rb") as file:
         encoded_string = urlsafe_b64encode(file.read()).decode("utf-8")
-    default_conf.update({'strategy': 'TestStrategy:{}'.format(encoded_string)})
+    default_conf.update({'strategy': 'SampleStrategy:{}'.format(encoded_string)})
 
     resolver = StrategyResolver(default_conf)
     assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'})
     # Make sure strategy was loaded from base64 (using temp directory)!!
-    assert log_has_re(r"Using resolved strategy TestStrategy from '"
-                      + tempfile.gettempdir() + r"/.*/TestStrategy\.py'\.\.\.", caplog)
+    assert log_has_re(r"Using resolved strategy SampleStrategy from '"
+                      + tempfile.gettempdir() + r"/.*/SampleStrategy\.py'\.\.\.", caplog)
 
 
 def test_load_strategy_invalid_directory(result, caplog, default_conf):
     resolver = StrategyResolver(default_conf)
     extra_dir = Path.cwd() / 'some/path'
-    resolver._load_strategy('TestStrategy', config=default_conf, extra_dir=extra_dir)
+    resolver._load_strategy('SampleStrategy', config=default_conf, extra_dir=extra_dir)
 
     assert log_has_re(r'Path .*' + r'some.*path.*' + r'.* does not exist', caplog)
 
@@ -96,16 +72,6 @@ def test_load_not_found_strategy(default_conf):
         strategy._load_strategy(strategy_name='NotFoundStrategy', config=default_conf)
 
 
-def test_load_staticmethod_importerror(mocker, caplog, default_conf):
-    mocker.patch("freqtrade.resolvers.strategy_resolver.import_strategy", Mock(
-        side_effect=TypeError("can't pickle staticmethod objects")))
-    with pytest.raises(OperationalException,
-                       match=r"Impossible to load Strategy 'DefaultStrategy'. "
-                             r"This class does not exist or contains Python code errors."):
-        StrategyResolver(default_conf)
-    assert log_has_re(r".*Error: can't pickle staticmethod objects", caplog)
-
-
 def test_strategy(result, default_conf):
     default_conf.update({'strategy': 'DefaultStrategy'})
 
@@ -380,6 +346,31 @@ def test_call_deprecated_function(result, monkeypatch, default_conf):
     assert resolver.strategy._populate_fun_len == 2
     assert resolver.strategy._buy_fun_len == 2
     assert resolver.strategy._sell_fun_len == 2
+    assert resolver.strategy.INTERFACE_VERSION == 1
+
+    indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata)
+    assert isinstance(indicator_df, DataFrame)
+    assert 'adx' in indicator_df.columns
+
+    buydf = resolver.strategy.advise_buy(result, metadata=metadata)
+    assert isinstance(buydf, DataFrame)
+    assert 'buy' in buydf.columns
+
+    selldf = resolver.strategy.advise_sell(result, metadata=metadata)
+    assert isinstance(selldf, DataFrame)
+    assert 'sell' in selldf
+
+
+def test_strategy_interface_versioning(result, monkeypatch, default_conf):
+    default_conf.update({'strategy': 'DefaultStrategy'})
+    resolver = StrategyResolver(default_conf)
+    metadata = {'pair': 'ETH/BTC'}
+
+    # Make sure we are using a legacy function
+    assert resolver.strategy._populate_fun_len == 3
+    assert resolver.strategy._buy_fun_len == 3
+    assert resolver.strategy._sell_fun_len == 3
+    assert resolver.strategy.INTERFACE_VERSION == 2
 
     indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata)
     assert isinstance(indicator_df, DataFrame)
diff --git a/tests/test_arguments.py b/tests/test_arguments.py
new file mode 100644
index 000000000..cf0104c01
--- /dev/null
+++ b/tests/test_arguments.py
@@ -0,0 +1,195 @@
+# pragma pylint: disable=missing-docstring, C0103
+import argparse
+
+import pytest
+
+from freqtrade.configuration import Arguments
+from freqtrade.configuration.cli_options import check_int_positive
+
+
+# Parse common command-line-arguments. Used for all tools
+def test_parse_args_none() -> None:
+    arguments = Arguments([])
+    assert isinstance(arguments, Arguments)
+    x = arguments.get_parsed_arg()
+    assert isinstance(x, dict)
+    assert isinstance(arguments.parser, argparse.ArgumentParser)
+
+
+def test_parse_args_defaults() -> None:
+    args = Arguments([]).get_parsed_arg()
+    assert args["config"] == ['config.json']
+    assert args["strategy_path"] is None
+    assert args["datadir"] is None
+    assert args["verbosity"] == 0
+
+
+def test_parse_args_config() -> None:
+    args = Arguments(['-c', '/dev/null']).get_parsed_arg()
+    assert args["config"] == ['/dev/null']
+
+    args = Arguments(['--config', '/dev/null']).get_parsed_arg()
+    assert args["config"] == ['/dev/null']
+
+    args = Arguments(['--config', '/dev/null',
+                      '--config', '/dev/zero'],).get_parsed_arg()
+    assert args["config"] == ['/dev/null', '/dev/zero']
+
+
+def test_parse_args_db_url() -> None:
+    args = Arguments(['--db-url', 'sqlite:///test.sqlite']).get_parsed_arg()
+    assert args["db_url"] == 'sqlite:///test.sqlite'
+
+
+def test_parse_args_verbose() -> None:
+    args = Arguments(['-v']).get_parsed_arg()
+    assert args["verbosity"] == 1
+
+    args = Arguments(['--verbose']).get_parsed_arg()
+    assert args["verbosity"] == 1
+
+
+def test_common_scripts_options() -> None:
+    args = Arguments(['download-data', '-p', 'ETH/BTC', 'XRP/BTC']).get_parsed_arg()
+
+    assert args["pairs"] == ['ETH/BTC', 'XRP/BTC']
+    assert "func" in args
+
+
+def test_parse_args_version() -> None:
+    with pytest.raises(SystemExit, match=r'0'):
+        Arguments(['--version']).get_parsed_arg()
+
+
+def test_parse_args_invalid() -> None:
+    with pytest.raises(SystemExit, match=r'2'):
+        Arguments(['-c']).get_parsed_arg()
+
+
+def test_parse_args_strategy() -> None:
+    args = Arguments(['--strategy', 'SomeStrategy']).get_parsed_arg()
+    assert args["strategy"] == 'SomeStrategy'
+
+
+def test_parse_args_strategy_invalid() -> None:
+    with pytest.raises(SystemExit, match=r'2'):
+        Arguments(['--strategy']).get_parsed_arg()
+
+
+def test_parse_args_strategy_path() -> None:
+    args = Arguments(['--strategy-path', '/some/path']).get_parsed_arg()
+    assert args["strategy_path"] == '/some/path'
+
+
+def test_parse_args_strategy_path_invalid() -> None:
+    with pytest.raises(SystemExit, match=r'2'):
+        Arguments(['--strategy-path']).get_parsed_arg()
+
+
+def test_parse_args_backtesting_invalid() -> None:
+    with pytest.raises(SystemExit, match=r'2'):
+        Arguments(['backtesting --ticker-interval']).get_parsed_arg()
+
+    with pytest.raises(SystemExit, match=r'2'):
+        Arguments(['backtesting --ticker-interval', 'abc']).get_parsed_arg()
+
+
+def test_parse_args_backtesting_custom() -> None:
+    args = [
+        '-c', 'test_conf.json',
+        'backtesting',
+        '--ticker-interval', '1m',
+        '--strategy-list',
+        'DefaultStrategy',
+        'SampleStrategy'
+        ]
+    call_args = Arguments(args).get_parsed_arg()
+    assert call_args["config"] == ['test_conf.json']
+    assert call_args["verbosity"] == 0
+    assert call_args["subparser"] == 'backtesting'
+    assert call_args["func"] is not None
+    assert call_args["ticker_interval"] == '1m'
+    assert type(call_args["strategy_list"]) is list
+    assert len(call_args["strategy_list"]) == 2
+
+
+def test_parse_args_hyperopt_custom() -> None:
+    args = [
+        '-c', 'test_conf.json',
+        'hyperopt',
+        '--epochs', '20',
+        '--spaces', 'buy'
+    ]
+    call_args = Arguments(args).get_parsed_arg()
+    assert call_args["config"] == ['test_conf.json']
+    assert call_args["epochs"] == 20
+    assert call_args["verbosity"] == 0
+    assert call_args["subparser"] == 'hyperopt'
+    assert call_args["spaces"] == ['buy']
+    assert call_args["func"] is not None
+    assert callable(call_args["func"])
+
+
+def test_download_data_options() -> None:
+    args = [
+        '--datadir', 'datadir/directory',
+        'download-data',
+        '--pairs-file', 'file_with_pairs',
+        '--days', '30',
+        '--exchange', 'binance'
+    ]
+    pargs = Arguments(args).get_parsed_arg()
+
+    assert pargs["pairs_file"] == 'file_with_pairs'
+    assert pargs["datadir"] == 'datadir/directory'
+    assert pargs["days"] == 30
+    assert pargs["exchange"] == 'binance'
+
+
+def test_plot_dataframe_options() -> None:
+    args = [
+        '-c', 'config.json.example',
+        'plot-dataframe',
+        '--indicators1', 'sma10', 'sma100',
+        '--indicators2', 'macd', 'fastd', 'fastk',
+        '--plot-limit', '30',
+        '-p', 'UNITTEST/BTC',
+    ]
+    pargs = Arguments(args).get_parsed_arg()
+
+    assert pargs["indicators1"] == ["sma10", "sma100"]
+    assert pargs["indicators2"] == ["macd", "fastd", "fastk"]
+    assert pargs["plot_limit"] == 30
+    assert pargs["pairs"] == ["UNITTEST/BTC"]
+
+
+def test_plot_profit_options() -> None:
+    args = [
+        'plot-profit',
+        '-p', 'UNITTEST/BTC',
+        '--trade-source', 'DB',
+        "--db-url", "sqlite:///whatever.sqlite",
+    ]
+    pargs = Arguments(args).get_parsed_arg()
+
+    assert pargs["trade_source"] == "DB"
+    assert pargs["pairs"] == ["UNITTEST/BTC"]
+    assert pargs["db_url"] == "sqlite:///whatever.sqlite"
+
+
+def test_check_int_positive() -> None:
+    assert check_int_positive("3") == 3
+    assert check_int_positive("1") == 1
+    assert check_int_positive("100") == 100
+
+    with pytest.raises(argparse.ArgumentTypeError):
+        check_int_positive("-2")
+
+    with pytest.raises(argparse.ArgumentTypeError):
+        check_int_positive("0")
+
+    with pytest.raises(argparse.ArgumentTypeError):
+        check_int_positive("3.5")
+
+    with pytest.raises(argparse.ArgumentTypeError):
+        check_int_positive("DeadBeef")
diff --git a/freqtrade/tests/test_configuration.py b/tests/test_configuration.py
similarity index 89%
rename from freqtrade/tests/test_configuration.py
rename to tests/test_configuration.py
index 153523f2a..330b82d39 100644
--- a/freqtrade/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -10,7 +10,8 @@ import pytest
 from jsonschema import Draft4Validator, ValidationError, validate
 
 from freqtrade import OperationalException, constants
-from freqtrade.configuration import Arguments, Configuration, validate_config_consistency
+from freqtrade.configuration import (Arguments, Configuration,
+                                     validate_config_consistency)
 from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.config_validation import validate_config_schema
 from freqtrade.configuration.directory_operations import (create_datadir,
@@ -19,14 +20,13 @@ from freqtrade.configuration.load_config import load_config_file
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
 from freqtrade.loggers import _set_loggers
 from freqtrade.state import RunMode
-from freqtrade.tests.conftest import (log_has, log_has_re,
-                                      patched_configuration_load_config_file)
+from tests.conftest import (log_has, log_has_re,
+                            patched_configuration_load_config_file)
 
 
 @pytest.fixture(scope="function")
 def all_conf():
-    config_file = Path(__file__).parents[2] / "config_full.json.example"
-    print(config_file)
+    config_file = Path(__file__).parents[1] / "config_full.json.example"
     conf = load_config_file(str(config_file))
     return conf
 
@@ -41,14 +41,14 @@ def test_load_config_invalid_pair(default_conf) -> None:
 def test_load_config_missing_attributes(default_conf) -> None:
     default_conf.pop('exchange')
 
-    with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'):
+    with pytest.raises(ValidationError, match=r".*'exchange' is a required property.*"):
         validate_config_schema(default_conf)
 
 
 def test_load_config_incorrect_stake_amount(default_conf) -> None:
     default_conf['stake_amount'] = 'fake'
 
-    with pytest.raises(ValidationError, match=r'.*\'fake\' does not match \'unlimited\'.*'):
+    with pytest.raises(ValidationError, match=r".*'fake' does not match 'unlimited'.*"):
         validate_config_schema(default_conf)
 
 
@@ -66,7 +66,7 @@ def test_load_config_file(default_conf, mocker, caplog) -> None:
 def test__args_to_config(caplog):
 
     arg_list = ['--strategy-path', 'TestTest']
-    args = Arguments(arg_list, '').get_parsed_arg()
+    args = Arguments(arg_list).get_parsed_arg()
     configuration = Configuration(args)
     config = {}
     with warnings.catch_warnings(record=True) as w:
@@ -93,7 +93,7 @@ def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None:
     default_conf['max_open_trades'] = 0
     patched_configuration_load_config_file(mocker, default_conf)
 
-    args = Arguments([], '').get_parsed_arg()
+    args = Arguments([]).get_parsed_arg()
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
 
@@ -119,7 +119,7 @@ def test_load_config_combine_dicts(default_conf, mocker, caplog) -> None:
     )
 
     arg_list = ['-c', 'test_conf.json', '--config', 'test2_conf.json', ]
-    args = Arguments(arg_list, '').get_parsed_arg()
+    args = Arguments(arg_list).get_parsed_arg()
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
 
@@ -143,12 +143,10 @@ def test_from_config(default_conf, mocker, caplog) -> None:
     conf2['exchange']['pair_whitelist'] += ['NANO/BTC']
     conf2['fiat_display_currency'] = "EUR"
     config_files = [conf1, conf2]
+    mocker.patch('freqtrade.configuration.configuration.create_datadir', lambda c, x: x)
 
     configsmock = MagicMock(side_effect=config_files)
-    mocker.patch(
-        'freqtrade.configuration.configuration.load_config_file',
-        configsmock
-    )
+    mocker.patch('freqtrade.configuration.configuration.load_config_file', configsmock)
 
     validated_conf = Configuration.from_files(['test_conf.json', 'test2_conf.json'])
 
@@ -161,13 +159,32 @@ def test_from_config(default_conf, mocker, caplog) -> None:
     assert validated_conf['fiat_display_currency'] == "EUR"
     assert 'internals' in validated_conf
     assert log_has('Validating configuration ...', caplog)
+    assert isinstance(validated_conf['user_data_dir'], Path)
+
+
+def test_print_config(default_conf, mocker, caplog) -> None:
+    conf1 = deepcopy(default_conf)
+    # Delete non-json elements from default_conf
+    del conf1['user_data_dir']
+    config_files = [conf1]
+
+    configsmock = MagicMock(side_effect=config_files)
+    mocker.patch('freqtrade.configuration.configuration.create_datadir', lambda c, x: x)
+    mocker.patch('freqtrade.configuration.configuration.load_config_file', configsmock)
+
+    validated_conf = Configuration.from_files(['test_conf.json'])
+
+    assert isinstance(validated_conf['user_data_dir'], Path)
+    assert "user_data_dir" in validated_conf
+    assert "original_config" in validated_conf
+    assert isinstance(json.dumps(validated_conf['original_config']), str)
 
 
 def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) -> None:
     default_conf['max_open_trades'] = -1
     patched_configuration_load_config_file(mocker, default_conf)
 
-    args = Arguments([], '').get_parsed_arg()
+    args = Arguments([]).get_parsed_arg()
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
 
@@ -191,7 +208,7 @@ def test_load_config_file_exception(mocker) -> None:
 def test_load_config(default_conf, mocker) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
 
-    args = Arguments([], '').get_parsed_arg()
+    args = Arguments([]).get_parsed_arg()
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
 
@@ -208,7 +225,7 @@ def test_load_config_with_params(default_conf, mocker) -> None:
         '--strategy-path', '/some/path',
         '--db-url', 'sqlite:///someurl',
     ]
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
 
@@ -226,7 +243,7 @@ def test_load_config_with_params(default_conf, mocker) -> None:
         '--strategy', 'TestStrategy',
         '--strategy-path', '/some/path'
     ]
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
@@ -242,7 +259,7 @@ def test_load_config_with_params(default_conf, mocker) -> None:
         '--strategy', 'TestStrategy',
         '--strategy-path', '/some/path'
     ]
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
@@ -258,7 +275,7 @@ def test_load_config_with_params(default_conf, mocker) -> None:
         '--strategy', 'TestStrategy',
         '--strategy-path', '/some/path'
     ]
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
@@ -276,7 +293,7 @@ def test_load_config_with_params(default_conf, mocker) -> None:
         '--strategy', 'TestStrategy',
         '--strategy-path', '/some/path'
     ]
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
@@ -290,7 +307,7 @@ def test_load_custom_strategy(default_conf, mocker) -> None:
     })
     patched_configuration_load_config_file(mocker, default_conf)
 
-    args = Arguments([], '').get_parsed_arg()
+    args = Arguments([]).get_parsed_arg()
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
 
@@ -305,7 +322,7 @@ def test_show_info(default_conf, mocker, caplog) -> None:
         '--strategy', 'TestStrategy',
         '--db-url', 'sqlite:///tmp/testdb',
     ]
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     configuration.get_config()
@@ -323,7 +340,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
         'backtesting'
     ]
 
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     config = configuration.get_config()
@@ -341,14 +358,10 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
     assert 'position_stacking' not in config
     assert not log_has('Parameter --enable-position-stacking detected ...', caplog)
 
-    assert 'refresh_pairs' not in config
-    assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog)
-
     assert 'timerange' not in config
     assert 'export' not in config
 
 
-@pytest.mark.filterwarnings("ignore:DEPRECATED")
 def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
     mocker.patch(
@@ -368,12 +381,11 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
         '--ticker-interval', '1m',
         '--enable-position-stacking',
         '--disable-max-market-positions',
-        '--refresh-pairs-cached',
         '--timerange', ':100',
         '--export', '/bar/foo'
     ]
 
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     config = configuration.get_config()
@@ -398,8 +410,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
     assert log_has('Parameter --disable-max-market-positions detected ...', caplog)
     assert log_has('max_open_trades set to unlimited ...', caplog)
 
-    assert 'refresh_pairs'in config
-    assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog)
     assert 'timerange' in config
     assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog)
 
@@ -423,7 +433,7 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non
         'TestStrategy'
     ]
 
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args, RunMode.BACKTEST)
     config = configuration.get_config()
@@ -440,7 +450,7 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non
                    caplog)
 
     assert 'strategy_list' in config
-    assert log_has('Using strategy list of 2 Strategies', caplog)
+    assert log_has('Using strategy list of 2 strategies', caplog)
 
     assert 'position_stacking' not in config
 
@@ -460,7 +470,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None:
         '--epochs', '10',
         '--spaces', 'all',
     ]
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args, RunMode.HYPEROPT)
     config = configuration.get_config()
@@ -472,13 +482,14 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None:
 
     assert 'spaces' in config
     assert config['spaces'] == ['all']
-    assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog)
+    assert log_has("Parameter -s/--spaces detected: ['all']", caplog)
     assert "runmode" in config
     assert config['runmode'] == RunMode.HYPEROPT
 
 
 def test_check_exchange(default_conf, caplog) -> None:
     # Test an officially supported by Freqtrade team exchange
+    default_conf['runmode'] = RunMode.DRY_RUN
     default_conf.get('exchange').update({'name': 'BITTREX'})
     assert check_exchange(default_conf)
     assert log_has_re(r"Exchange .* is officially supported by the Freqtrade development team\.",
@@ -493,7 +504,7 @@ def test_check_exchange(default_conf, caplog) -> None:
     caplog.clear()
 
     # Test an available exchange, supported by ccxt
-    default_conf.get('exchange').update({'name': 'kraken'})
+    default_conf.get('exchange').update({'name': 'huobipro'})
     assert check_exchange(default_conf)
     assert log_has_re(r"Exchange .* is supported by ccxt and .* not officially supported "
                       r"by the Freqtrade development team\. .*", caplog)
@@ -523,6 +534,18 @@ def test_check_exchange(default_conf, caplog) -> None:
     ):
         check_exchange(default_conf)
 
+    # Test no exchange...
+    default_conf.get('exchange').update({'name': ''})
+    default_conf['runmode'] = RunMode.PLOT
+    assert check_exchange(default_conf)
+
+    # Test no exchange...
+    default_conf.get('exchange').update({'name': ''})
+    default_conf['runmode'] = RunMode.OTHER
+    with pytest.raises(OperationalException,
+                       match=r'This command requires a configured exchange.*'):
+        check_exchange(default_conf)
+
 
 def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
@@ -530,7 +553,7 @@ def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None:
     # Prevent setting loggers
     mocker.patch('freqtrade.loggers._set_loggers', MagicMock)
     arglist = ['-vvv']
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
@@ -583,7 +606,7 @@ def test_set_logfile(default_conf, mocker):
     arglist = [
         '--logfile', 'test_file.log',
     ]
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
 
@@ -597,7 +620,7 @@ def test_load_config_warn_forcebuy(default_conf, mocker, caplog) -> None:
     default_conf['forcebuy_enable'] = True
     patched_configuration_load_config_file(mocker, default_conf)
 
-    args = Arguments([], '').get_parsed_arg()
+    args = Arguments([]).get_parsed_arg()
     configuration = Configuration(args)
     validated_conf = configuration.load_config()
 
@@ -706,7 +729,6 @@ def test_load_config_test_comments() -> None:
     Load config with comments
     """
     config_file = Path(__file__).parents[0] / "config_test_comments.json"
-    print(config_file)
     conf = load_config_file(str(config_file))
 
     assert conf
@@ -722,7 +744,7 @@ def test_load_config_default_exchange(all_conf) -> None:
     assert 'exchange' not in all_conf
 
     with pytest.raises(ValidationError,
-                       match=r'\'exchange\' is a required property'):
+                       match=r"'exchange' is a required property"):
         validate_config_schema(all_conf)
 
 
@@ -736,7 +758,7 @@ def test_load_config_default_exchange_name(all_conf) -> None:
     assert 'name' not in all_conf['exchange']
 
     with pytest.raises(ValidationError,
-                       match=r'\'name\' is a required property'):
+                       match=r"'name' is a required property"):
         validate_config_schema(all_conf)
 
 
@@ -772,7 +794,7 @@ def test_pairlist_resolving():
         '--exchange', 'binance'
     ]
 
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     config = configuration.get_config()
@@ -788,7 +810,7 @@ def test_pairlist_resolving_with_config(mocker, default_conf):
         'download-data',
     ]
 
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     config = configuration.get_config()
@@ -803,7 +825,7 @@ def test_pairlist_resolving_with_config(mocker, default_conf):
         '--pairs', 'ETH/BTC', 'XRP/BTC',
     ]
 
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     config = configuration.get_config()
@@ -825,7 +847,7 @@ def test_pairlist_resolving_with_config_pl(mocker, default_conf):
         '--pairs-file', 'pairs.json',
     ]
 
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     config = configuration.get_config()
@@ -847,7 +869,7 @@ def test_pairlist_resolving_with_config_pl_not_exists(mocker, default_conf):
         '--pairs-file', 'pairs.json',
     ]
 
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
 
     with pytest.raises(OperationalException, match=r"No pairs file found with path.*"):
         configuration = Configuration(args)
@@ -864,7 +886,9 @@ def test_pairlist_resolving_fallback(mocker):
         '--exchange', 'binance'
     ]
 
-    args = Arguments(arglist, '').get_parsed_arg()
+    args = Arguments(arglist).get_parsed_arg()
+    # Fix flaky tests if config.json exists
+    args["config"] = None
 
     configuration = Configuration(args)
     config = configuration.get_config()
diff --git a/freqtrade/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
similarity index 93%
rename from freqtrade/tests/test_freqtradebot.py
rename to tests/test_freqtradebot.py
index dab7a9ff7..ee28f2e58 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -4,6 +4,7 @@
 import logging
 import time
 from copy import deepcopy
+from math import isclose
 from unittest.mock import MagicMock, PropertyMock
 
 import arrow
@@ -12,17 +13,17 @@ import requests
 
 from freqtrade import (DependencyException, InvalidOrderException,
                        OperationalException, TemporaryError, constants)
+from freqtrade.constants import MATH_CLOSE_PREC
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.persistence import Trade
 from freqtrade.rpc import RPCMessageType
-from freqtrade.state import State, RunMode
+from freqtrade.state import RunMode, State
 from freqtrade.strategy.interface import SellCheckTuple, SellType
-from freqtrade.tests.conftest import (get_patched_freqtradebot,
-                                      get_patched_worker, log_has, log_has_re,
-                                      patch_edge, patch_exchange,
-                                      patch_get_signal, patch_wallet)
 from freqtrade.worker import Worker
+from tests.conftest import (get_patched_freqtradebot, get_patched_worker,
+                            log_has, log_has_re, patch_edge, patch_exchange,
+                            patch_get_signal, patch_wallet)
 
 
 def patch_RPCManager(mocker) -> MagicMock:
@@ -147,8 +148,7 @@ def test_order_dict_dry_run(default_conf, mocker, caplog) -> None:
     }
 
     freqtrade = FreqtradeBot(conf)
-    assert log_has("Disabling stoploss_on_exchange during dry-run.", caplog)
-    assert not freqtrade.strategy.order_types['stoploss_on_exchange']
+    assert freqtrade.strategy.order_types['stoploss_on_exchange']
 
     caplog.clear()
     # is left untouched
@@ -972,7 +972,6 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non
         markets=PropertyMock(return_value=markets)
     )
     pair = 'ETH/BTC'
-    print(buy_mm.call_args_list)
 
     assert freqtrade.execute_buy(pair, stake_amount)
     assert get_bid.call_count == 1
@@ -1153,7 +1152,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
         side_effect=DependencyException()
     )
     freqtrade.handle_stoploss_on_exchange(trade)
-    assert log_has('Unable to place a stoploss order on exchange: ', caplog)
+    assert log_has('Unable to place a stoploss order on exchange.', caplog)
     assert trade.stoploss_order_id is None
 
     # Fifth case: get_order returns InvalidOrder
@@ -1201,6 +1200,50 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog,
     assert trade.is_open is True
 
 
+def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee,
+                                             markets, limit_buy_order, limit_sell_order):
+    rpc_mock = patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    sell_mock = MagicMock(return_value={'id': limit_sell_order['id']})
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        get_ticker=MagicMock(return_value={
+            'bid': 0.00001172,
+            'ask': 0.00001173,
+            'last': 0.00001172
+        }),
+        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        sell=sell_mock,
+        get_fee=fee,
+        markets=PropertyMock(return_value=markets),
+        get_order=MagicMock(return_value={'status': 'canceled'}),
+        stoploss_limit=MagicMock(side_effect=InvalidOrderException()),
+    )
+    freqtrade = FreqtradeBot(default_conf)
+    patch_get_signal(freqtrade)
+    freqtrade.strategy.order_types['stoploss_on_exchange'] = True
+
+    freqtrade.create_trades()
+    trade = Trade.query.first()
+    caplog.clear()
+    freqtrade.create_stoploss_order(trade, 200, 199)
+    assert trade.stoploss_order_id is None
+    assert trade.sell_reason == SellType.EMERGENCY_SELL.value
+    assert log_has("Unable to place a stoploss order on exchange. ", caplog)
+    assert log_has("Selling the trade forcefully", caplog)
+
+    # Should call a market sell
+    assert sell_mock.call_count == 1
+    assert sell_mock.call_args[1]['ordertype'] == 'market'
+    assert sell_mock.call_args[1]['pair'] == trade.pair
+    assert sell_mock.call_args[1]['amount'] == trade.amount
+
+    # Rpc is sending first buy, then sell
+    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_args_list[1][0][0]['sell_reason'] == SellType.EMERGENCY_SELL.value
+    assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market'
+
+
 def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog,
                                               markets, limit_buy_order, limit_sell_order) -> None:
     # When trailing stoploss is set
@@ -1549,6 +1592,8 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No
     Trade.session = MagicMock()
     trade.open_order_id = '123'
     trade.open_fee = 0.001
+    # Add datetime explicitly since sqlalchemy defaults apply only once written to database
+    trade.open_date = arrow.utcnow().datetime
     freqtrade.update_trade_state(trade)
     # Test amount not modified by fee-logic
     assert not log_has_re(r'Applying fee to .*', caplog)
@@ -1592,6 +1637,31 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_
     assert trade.amount == limit_buy_order['amount']
 
 
+def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_order,
+                                                       limit_buy_order, mocker, caplog):
+    trades_for_order[0]['amount'] = limit_buy_order['amount'] + 1e-14
+    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
+    # get_order should not be called!!
+    mocker.patch('freqtrade.exchange.Exchange.get_order', MagicMock(side_effect=ValueError))
+    patch_exchange(mocker)
+    Trade.session = MagicMock()
+    amount = sum(x['amount'] for x in trades_for_order)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    trade = Trade(
+        pair='LTC/ETH',
+        amount=amount,
+        exchange='binance',
+        open_rate=0.245441,
+        open_order_id="123456",
+        is_open=True,
+        open_date=arrow.utcnow().datetime,
+    )
+    freqtrade.update_trade_state(trade, limit_buy_order)
+    assert trade.amount != amount
+    assert trade.amount == limit_buy_order['amount']
+    assert log_has_re(r'Applying fee on amount for .*', caplog)
+
+
 def test_update_trade_state_exception(mocker, default_conf,
                                       limit_buy_order, caplog) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
@@ -1782,7 +1852,8 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order,
     # if ROI is reached we must sell
     patch_get_signal(freqtrade, value=(False, True))
     assert freqtrade.handle_trade(trade)
-    assert log_has('Required profit reached. Selling..', caplog)
+    assert log_has("ETH/BTC - Required profit reached. sell_flag=True, sell_type=SellType.ROI",
+                   caplog)
 
 
 def test_handle_trade_experimental(
@@ -1812,7 +1883,8 @@ def test_handle_trade_experimental(
 
     patch_get_signal(freqtrade, value=(False, True))
     assert freqtrade.handle_trade(trade)
-    assert log_has('Sell signal received. Selling..', caplog)
+    assert log_has("ETH/BTC - Sell signal received. sell_flag=True, sell_type=SellType.SELL_SIGNAL",
+                   caplog)
 
 
 def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order,
@@ -2091,6 +2163,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) -
     )
     freqtrade = FreqtradeBot(default_conf)
 
+    open_date = arrow.utcnow().shift(minutes=-601)
     trade_buy = Trade(
         pair='ETH/BTC',
         open_rate=0.00001099,
@@ -2100,16 +2173,18 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) -
         fee_open=0.0,
         fee_close=0.0,
         stake_amount=1,
-        open_date=arrow.utcnow().shift(minutes=-601).datetime,
+        open_date=open_date.datetime,
         is_open=True
     )
 
     Trade.session.add(trade_buy)
 
     freqtrade.check_handle_timedout()
-    assert log_has_re(r'Cannot query order for Trade\(id=1, pair=ETH/BTC, amount=90.99181073, '
-                      r'open_rate=0.00001099, open_since=10 hours ago\) due to Traceback \(most '
-                      r'recent call last\):\n.*', caplog)
+    assert log_has_re(r"Cannot query order for Trade\(id=1, pair=ETH/BTC, amount=90.99181073, "
+                      r"open_rate=0.00001099, open_since="
+                      f"{open_date.strftime('%Y-%m-%d %H:%M:%S')}"
+                      r"\) due to Traceback \(most recent call last\):\n*",
+                      caplog)
 
 
 def test_handle_timedout_limit_buy(mocker, default_conf) -> None:
@@ -2159,9 +2234,9 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None:
 
 def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
+    patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        _load_markets=MagicMock(return_value={}),
         get_ticker=ticker,
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
@@ -2205,9 +2280,9 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc
 
 def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
+    patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        _load_markets=MagicMock(return_value={}),
         get_ticker=ticker,
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
@@ -2254,9 +2329,9 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe
                                                         ticker_sell_down,
                                                         markets, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
+    patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        _load_markets=MagicMock(return_value={}),
         get_ticker=ticker,
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
@@ -2311,9 +2386,9 @@ def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee,
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException())
     sellmock = MagicMock()
+    patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        _load_markets=MagicMock(return_value={}),
         get_ticker=ticker,
         get_fee=fee,
         markets=PropertyMock(return_value=markets),
@@ -2342,9 +2417,9 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf,
 
     default_conf['exchange']['name'] = 'binance'
     rpc_mock = patch_RPCManager(mocker)
+    patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        _load_markets=MagicMock(return_value={}),
         get_ticker=ticker,
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
@@ -2397,9 +2472,9 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf,
                                                          markets, mocker) -> None:
     default_conf['exchange']['name'] = 'binance'
     rpc_mock = patch_RPCManager(mocker)
+    patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        _load_markets=MagicMock(return_value={}),
         get_ticker=ticker,
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
@@ -2414,7 +2489,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf,
 
     mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y)
     mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit)
+    mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit)
 
     freqtrade = FreqtradeBot(default_conf)
     freqtrade.strategy.order_types['stoploss_on_exchange'] = True
@@ -2454,7 +2529,6 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf,
     freqtrade.process_maybe_execute_sell(trade)
     assert trade.stoploss_order_id is None
     assert trade.is_open is False
-    print(trade.sell_reason)
     assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value
     assert rpc_mock.call_count == 2
 
@@ -2462,9 +2536,9 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf,
 def test_execute_sell_market_order(default_conf, ticker, fee,
                                    ticker_sell_up, markets, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
+    patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        _load_markets=MagicMock(return_value={}),
         get_ticker=ticker,
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
@@ -2638,9 +2712,9 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke
 
 def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, markets, mocker, caplog) -> None:
     patch_RPCManager(mocker)
+    patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        _load_markets=MagicMock(return_value={}),
         get_ticker=ticker,
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
@@ -2753,8 +2827,9 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog,
     # Sell as trailing-stop is reached
     assert freqtrade.handle_trade(trade) is True
     assert log_has(
-        f'HIT STOP: current price at 0.000012, stop loss is 0.000015, '
-        f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog)
+        f"ETH/BTC - HIT STOP: current price at 0.000012, "
+        f"stoploss is 0.000015, "
+        f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog)
     assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value
 
 
@@ -2796,8 +2871,8 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets
                  }))
     # stop-loss not reached, adjusted stoploss
     assert freqtrade.handle_trade(trade) is False
-    assert log_has(f'using positive stop loss: 0.01 offset: 0 profit: 0.2666%', caplog)
-    assert log_has(f'adjusted stop loss', caplog)
+    assert log_has(f"ETH/BTC - Using positive stoploss: 0.01 offset: 0 profit: 0.2666%", caplog)
+    assert log_has(f"ETH/BTC - Adjusting stoploss...", caplog)
     assert trade.stop_loss == 0.0000138501
 
     mocker.patch('freqtrade.exchange.Exchange.get_ticker',
@@ -2809,9 +2884,9 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets
     # Lower price again (but still positive)
     assert freqtrade.handle_trade(trade) is True
     assert log_has(
-        f'HIT STOP: current price at {buy_price + 0.000002:.6f}, '
-        f'stop loss is {trade.stop_loss:.6f}, '
-        f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog)
+        f"ETH/BTC - HIT STOP: current price at {buy_price + 0.000002:.6f}, "
+        f"stoploss is {trade.stop_loss:.6f}, "
+        f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog)
 
 
 def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee,
@@ -2854,8 +2929,9 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee,
                  }))
     # stop-loss not reached, adjusted stoploss
     assert freqtrade.handle_trade(trade) is False
-    assert log_has(f'using positive stop loss: 0.01 offset: 0.011 profit: 0.2666%', caplog)
-    assert log_has(f'adjusted stop loss', caplog)
+    assert log_has(f"ETH/BTC - Using positive stoploss: 0.01 offset: 0.011 profit: 0.2666%",
+                   caplog)
+    assert log_has(f"ETH/BTC - Adjusting stoploss...", caplog)
     assert trade.stop_loss == 0.0000138501
 
     mocker.patch('freqtrade.exchange.Exchange.get_ticker',
@@ -2867,9 +2943,9 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee,
     # Lower price again (but still positive)
     assert freqtrade.handle_trade(trade) is True
     assert log_has(
-        f'HIT STOP: current price at {buy_price + 0.000002:.6f}, '
-        f'stop loss is {trade.stop_loss:.6f}, '
-        f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog)
+        f"ETH/BTC - HIT STOP: current price at {buy_price + 0.000002:.6f}, "
+        f"stoploss is {trade.stop_loss:.6f}, "
+        f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog)
     assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value
 
 
@@ -2920,7 +2996,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee,
     # stop-loss should not be adjusted as offset is not reached yet
     assert freqtrade.handle_trade(trade) is False
 
-    assert not log_has(f'adjusted stop loss', caplog)
+    assert not log_has(f"ETH/BTC - Adjusting stoploss...", caplog)
     assert trade.stop_loss == 0.0000098910
 
     # price rises above the offset (rises 12% when the offset is 5.5%)
@@ -2932,8 +3008,9 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee,
                  }))
 
     assert freqtrade.handle_trade(trade) is False
-    assert log_has(f'using positive stop loss: 0.05 offset: 0.055 profit: 0.1218%', caplog)
-    assert log_has(f'adjusted stop loss', caplog)
+    assert log_has(f"ETH/BTC - Using positive stoploss: 0.05 offset: 0.055 profit: 0.1218%",
+                   caplog)
+    assert log_has(f"ETH/BTC - Adjusting stoploss...", caplog)
     assert trade.stop_loss == 0.0000117705
 
 
@@ -3155,6 +3232,54 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order
     assert freqtrade.get_real_amount(trade, limit_buy_order) == amount
 
 
+def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_fee, mocker):
+    limit_buy_order = deepcopy(buy_order_fee)
+    limit_buy_order['amount'] = limit_buy_order['amount'] - 0.001
+
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
+    amount = float(sum(x['amount'] for x in trades_for_order))
+    trade = Trade(
+        pair='LTC/ETH',
+        amount=amount,
+        exchange='binance',
+        open_rate=0.245441,
+        open_order_id="123456"
+    )
+    freqtrade = FreqtradeBot(default_conf)
+    patch_get_signal(freqtrade)
+
+    # Amount does not change
+    with pytest.raises(OperationalException, match=r"Half bought\? Amounts don't match"):
+        freqtrade.get_real_amount(trade, limit_buy_order)
+
+
+def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, buy_order_fee,
+                                               mocker):
+    # Floats should not be compared directly.
+    limit_buy_order = deepcopy(buy_order_fee)
+    trades_for_order[0]['amount'] = trades_for_order[0]['amount'] + 1e-15
+
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
+    amount = float(sum(x['amount'] for x in trades_for_order))
+    trade = Trade(
+        pair='LTC/ETH',
+        amount=amount,
+        exchange='binance',
+        open_rate=0.245441,
+        open_order_id="123456"
+    )
+    freqtrade = FreqtradeBot(default_conf)
+    patch_get_signal(freqtrade)
+
+    # Amount changes by fee amount.
+    assert isclose(freqtrade.get_real_amount(trade, limit_buy_order), amount - (amount * 0.001),
+                   abs_tol=MATH_CLOSE_PREC,)
+
+
 def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, mocker):
     # Remove "Currency" from fee dict
     trades_for_order[0]['fee'] = {'cost': 0.008}
@@ -3302,8 +3427,8 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets)
     default_conf['telegram']['enabled'] = False
 
     freqtrade = FreqtradeBot(default_conf)
-    # ordrebook shall be used even if tickers would be lower.
-    assert freqtrade.get_target_bid('ETH/BTC', ) != 0.042
+    # orderbook shall be used even if tickers would be lower.
+    assert freqtrade.get_target_bid('ETH/BTC') != 0.042
     assert ticker_mock.call_count == 0
 
 
diff --git a/freqtrade/tests/test_indicator_helpers.py b/tests/test_indicator_helpers.py
similarity index 100%
rename from freqtrade/tests/test_indicator_helpers.py
rename to tests/test_indicator_helpers.py
diff --git a/freqtrade/tests/test_main.py b/tests/test_main.py
similarity index 88%
rename from freqtrade/tests/test_main.py
rename to tests/test_main.py
index db5a438d0..d73edc0da 100644
--- a/freqtrade/tests/test_main.py
+++ b/tests/test_main.py
@@ -10,9 +10,9 @@ from freqtrade.configuration import Arguments
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.main import main
 from freqtrade.state import State
-from freqtrade.tests.conftest import (log_has, patch_exchange,
-                                      patched_configuration_load_config_file)
 from freqtrade.worker import Worker
+from tests.conftest import (log_has, patch_exchange,
+                            patched_configuration_load_config_file)
 
 
 def test_parse_args_backtesting(mocker) -> None:
@@ -27,11 +27,12 @@ def test_parse_args_backtesting(mocker) -> None:
         main(['backtesting'])
     assert backtesting_mock.call_count == 1
     call_args = backtesting_mock.call_args[0][0]
-    assert call_args.config == ['config.json']
-    assert call_args.verbosity == 0
-    assert call_args.subparser == 'backtesting'
-    assert call_args.func is not None
-    assert call_args.ticker_interval is None
+    assert call_args["config"] == ['config.json']
+    assert call_args["verbosity"] == 0
+    assert call_args["subparser"] == 'backtesting'
+    assert call_args["func"] is not None
+    assert callable(call_args["func"])
+    assert call_args["ticker_interval"] is None
 
 
 def test_main_start_hyperopt(mocker) -> None:
@@ -42,10 +43,11 @@ def test_main_start_hyperopt(mocker) -> None:
         main(['hyperopt'])
     assert hyperopt_mock.call_count == 1
     call_args = hyperopt_mock.call_args[0][0]
-    assert call_args.config == ['config.json']
-    assert call_args.verbosity == 0
-    assert call_args.subparser == 'hyperopt'
-    assert call_args.func is not None
+    assert call_args["config"] == ['config.json']
+    assert call_args["verbosity"] == 0
+    assert call_args["subparser"] == 'hyperopt'
+    assert call_args["func"] is not None
+    assert callable(call_args["func"])
 
 
 def test_main_fatal_exception(mocker, default_conf, caplog) -> None:
@@ -117,7 +119,7 @@ def test_main_reload_conf(mocker, default_conf, caplog) -> None:
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
     mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
 
-    args = Arguments(['-c', 'config.json.example'], '').get_parsed_arg()
+    args = Arguments(['-c', 'config.json.example']).get_parsed_arg()
     worker = Worker(args=args, config=default_conf)
     with pytest.raises(SystemExit):
         main(['-c', 'config.json.example'])
@@ -139,7 +141,7 @@ def test_reconfigure(mocker, default_conf) -> None:
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
     mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
 
-    args = Arguments(['-c', 'config.json.example'], '').get_parsed_arg()
+    args = Arguments(['-c', 'config.json.example']).get_parsed_arg()
     worker = Worker(args=args, config=default_conf)
     freqtrade = worker.freqtrade
 
diff --git a/freqtrade/tests/test_misc.py b/tests/test_misc.py
similarity index 89%
rename from freqtrade/tests/test_misc.py
rename to tests/test_misc.py
index c55083e64..320ed208c 100644
--- a/freqtrade/tests/test_misc.py
+++ b/tests/test_misc.py
@@ -45,16 +45,16 @@ def test_file_dump_json(mocker) -> None:
     assert json_dump.call_count == 1
 
 
-def test_file_load_json(mocker) -> None:
+def test_file_load_json(mocker, testdatadir) -> None:
 
     # 7m .json does not exist
-    ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '7m'))
+    ret = file_load_json(pair_data_filename(testdatadir, 'UNITTEST/BTC', '7m'))
     assert not ret
     # 1m json exists (but no .gz exists)
-    ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '1m'))
+    ret = file_load_json(pair_data_filename(testdatadir, 'UNITTEST/BTC', '1m'))
     assert ret
     # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json
-    ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '8m'))
+    ret = file_load_json(pair_data_filename(testdatadir, 'UNITTEST/BTC', '8m'))
     assert ret
 
 
diff --git a/freqtrade/tests/test_persistence.py b/tests/test_persistence.py
similarity index 99%
rename from freqtrade/tests/test_persistence.py
rename to tests/test_persistence.py
index c3ab7c128..6bd223a9b 100644
--- a/freqtrade/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -8,7 +8,7 @@ from sqlalchemy import create_engine
 
 from freqtrade import OperationalException, constants
 from freqtrade.persistence import Trade, clean_dry_run_db, init
-from freqtrade.tests.conftest import log_has
+from tests.conftest import log_has
 
 
 def create_mock_trades(fee):
@@ -729,7 +729,6 @@ def test_to_json(default_conf, fee):
     )
     result = trade.to_json()
     assert isinstance(result, dict)
-    print(result)
 
     assert result == {'trade_id': None,
                       'pair': 'ETH/BTC',
diff --git a/freqtrade/tests/test_plotting.py b/tests/test_plotting.py
similarity index 65%
rename from freqtrade/tests/test_plotting.py
rename to tests/test_plotting.py
index a78e38c1f..9028ab961 100644
--- a/freqtrade/tests/test_plotting.py
+++ b/tests/test_plotting.py
@@ -4,18 +4,22 @@ from pathlib import Path
 from unittest.mock import MagicMock
 
 import plotly.graph_objects as go
+import pytest
 from plotly.subplots import make_subplots
 
+from freqtrade import OperationalException
 from freqtrade.configuration import TimeRange
 from freqtrade.data import history
 from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data
+from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit
 from freqtrade.plot.plotting import (add_indicators, add_profit,
+                                     load_and_plot_trades,
                                      generate_candlestick_graph,
                                      generate_plot_filename,
                                      generate_profit_graph, init_plotscript,
-                                     plot_trades, store_plot_file)
+                                     plot_profit, plot_trades, store_plot_file)
 from freqtrade.strategy.default_strategy import DefaultStrategy
-from freqtrade.tests.conftest import log_has, log_has_re
+from tests.conftest import get_args, log_has, log_has_re
 
 
 def fig_generating_mock(fig, *args, **kwargs):
@@ -28,7 +32,7 @@ def find_trace_in_fig_data(data, search_string: str):
     return next(matches)
 
 
-def generage_empty_figure():
+def generate_empty_figure():
     return make_subplots(
         rows=3,
         cols=1,
@@ -38,18 +42,16 @@ def generage_empty_figure():
     )
 
 
-def test_init_plotscript(default_conf, mocker):
+def test_init_plotscript(default_conf, mocker, testdatadir):
     default_conf['timerange'] = "20180110-20180112"
     default_conf['trade_source'] = "file"
     default_conf['ticker_interval'] = "5m"
-    default_conf["datadir"] = history.make_testdata_path(None)
-    default_conf['exportfilename'] = str(
-        history.make_testdata_path(None) / "backtest-result_test.json")
+    default_conf["datadir"] = testdatadir
+    default_conf['exportfilename'] = str(testdatadir / "backtest-result_test.json")
     ret = init_plotscript(default_conf)
     assert "tickers" in ret
     assert "trades" in ret
     assert "pairs" in ret
-    assert "strategy" in ret
 
     default_conf['pairs'] = ["POWR/BTC", "XLM/BTC"]
     ret = init_plotscript(default_conf)
@@ -58,19 +60,19 @@ def test_init_plotscript(default_conf, mocker):
     assert "XLM/BTC" in ret["tickers"]
 
 
-def test_add_indicators(default_conf, caplog):
+def test_add_indicators(default_conf, testdatadir, caplog):
     pair = "UNITTEST/BTC"
     timerange = TimeRange(None, 'line', 0, -1000)
 
     data = history.load_pair_history(pair=pair, ticker_interval='1m',
-                                     datadir=None, timerange=timerange)
+                                     datadir=testdatadir, timerange=timerange)
     indicators1 = ["ema10"]
     indicators2 = ["macd"]
 
     # Generate buy/sell signals and indicators
     strat = DefaultStrategy(default_conf)
     data = strat.analyze_ticker(data, {'pair': pair})
-    fig = generage_empty_figure()
+    fig = generate_empty_figure()
 
     # Row 1
     fig1 = add_indicators(fig=deepcopy(fig), row=1, indicators=indicators1, data=data)
@@ -91,14 +93,14 @@ def test_add_indicators(default_conf, caplog):
     assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog)
 
 
-def test_plot_trades(caplog):
-    fig1 = generage_empty_figure()
+def test_plot_trades(testdatadir, caplog):
+    fig1 = generate_empty_figure()
     # nothing happens when no trades are available
     fig = plot_trades(fig1, None)
     assert fig == fig1
     assert log_has("No trades found.", caplog)
     pair = "ADA/BTC"
-    filename = history.make_testdata_path(None) / "backtest-result_test.json"
+    filename = testdatadir / "backtest-result_test.json"
     trades = load_backtest_data(filename)
     trades = trades.loc[trades['pair'] == pair]
 
@@ -119,7 +121,7 @@ def test_plot_trades(caplog):
     assert trade_sell.marker.color == 'red'
 
 
-def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, caplog):
+def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, testdatadir, caplog):
     row_mock = mocker.patch('freqtrade.plot.plotting.add_indicators',
                             MagicMock(side_effect=fig_generating_mock))
     trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades',
@@ -128,7 +130,7 @@ def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, c
     pair = "UNITTEST/BTC"
     timerange = TimeRange(None, 'line', 0, -1000)
     data = history.load_pair_history(pair=pair, ticker_interval='1m',
-                                     datadir=None, timerange=timerange)
+                                     datadir=testdatadir, timerange=timerange)
     data['buy'] = 0
     data['sell'] = 0
 
@@ -155,7 +157,7 @@ def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, c
     assert log_has("No sell-signals found.", caplog)
 
 
-def test_generate_candlestick_graph_no_trades(default_conf, mocker):
+def test_generate_candlestick_graph_no_trades(default_conf, mocker, testdatadir):
     row_mock = mocker.patch('freqtrade.plot.plotting.add_indicators',
                             MagicMock(side_effect=fig_generating_mock))
     trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades',
@@ -163,7 +165,7 @@ def test_generate_candlestick_graph_no_trades(default_conf, mocker):
     pair = 'UNITTEST/BTC'
     timerange = TimeRange(None, 'line', 0, -1000)
     data = history.load_pair_history(pair=pair, ticker_interval='1m',
-                                     datadir=None, timerange=timerange)
+                                     datadir=testdatadir, timerange=timerange)
 
     # Generate buy/sell signals and indicators
     strat = DefaultStrategy(default_conf)
@@ -208,7 +210,7 @@ def test_generate_Plot_filename():
 
 
 def test_generate_plot_file(mocker, caplog):
-    fig = generage_empty_figure()
+    fig = generate_empty_figure()
     plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock())
     store_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html",
                     directory=Path("user_data/plots"))
@@ -221,14 +223,14 @@ def test_generate_plot_file(mocker, caplog):
                    caplog)
 
 
-def test_add_profit():
-    filename = history.make_testdata_path(None) / "backtest-result_test.json"
+def test_add_profit(testdatadir):
+    filename = testdatadir / "backtest-result_test.json"
     bt_data = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
     df = history.load_pair_history(pair="POWR/BTC", ticker_interval='5m',
-                                   datadir=None, timerange=timerange)
-    fig = generage_empty_figure()
+                                   datadir=testdatadir, timerange=timerange)
+    fig = generate_empty_figure()
 
     cum_profits = create_cum_profit(df.set_index('date'),
                                     bt_data[bt_data["pair"] == 'POWR/BTC'],
@@ -241,13 +243,13 @@ def test_add_profit():
     assert profits.yaxis == "y2"
 
 
-def test_generate_profit_graph():
-    filename = history.make_testdata_path(None) / "backtest-result_test.json"
+def test_generate_profit_graph(testdatadir):
+    filename = testdatadir / "backtest-result_test.json"
     trades = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
     pairs = ["POWR/BTC", "XLM/BTC"]
 
-    tickers = history.load_data(datadir=None,
+    tickers = history.load_data(datadir=testdatadir,
                                 pairs=pairs,
                                 ticker_interval='5m',
                                 timerange=timerange
@@ -257,7 +259,11 @@ def test_generate_profit_graph():
     fig = generate_profit_graph(pairs, tickers, trades)
     assert isinstance(fig, go.Figure)
 
-    assert fig.layout.title.text == "Profit plot"
+    assert fig.layout.title.text == "Freqtrade Profit plot"
+    assert fig.layout.yaxis.title.text == "Price"
+    assert fig.layout.yaxis2.title.text == "Profit"
+    assert fig.layout.yaxis3.title.text == "Profit"
+
     figure = fig.layout.figure
     assert len(figure.data) == 4
 
@@ -270,3 +276,97 @@ def test_generate_profit_graph():
     for pair in pairs:
         profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}")
         assert isinstance(profit_pair, go.Scattergl)
+
+
+def test_start_plot_dataframe(mocker):
+    aup = mocker.patch("freqtrade.plot.plotting.load_and_plot_trades", MagicMock())
+    args = [
+        "--config", "config.json.example",
+        "plot-dataframe",
+        "--pairs", "ETH/BTC"
+    ]
+    start_plot_dataframe(get_args(args))
+
+    assert aup.call_count == 1
+    called_config = aup.call_args_list[0][0][0]
+    assert "pairs" in called_config
+    assert called_config['pairs'] == ["ETH/BTC"]
+
+
+def test_load_and_plot_trades(default_conf, mocker, caplog, testdatadir):
+    default_conf['trade_source'] = 'file'
+    default_conf["datadir"] = testdatadir
+    default_conf['exportfilename'] = str(testdatadir / "backtest-result_test.json")
+    default_conf['indicators1'] = ["sma5", "ema10"]
+    default_conf['indicators2'] = ["macd"]
+    default_conf['pairs'] = ["ETH/BTC", "LTC/BTC"]
+
+    candle_mock = MagicMock()
+    store_mock = MagicMock()
+    mocker.patch.multiple(
+        "freqtrade.plot.plotting",
+        generate_candlestick_graph=candle_mock,
+        store_plot_file=store_mock
+        )
+    load_and_plot_trades(default_conf)
+
+    # Both mocks should be called once per pair
+    assert candle_mock.call_count == 2
+    assert store_mock.call_count == 2
+
+    assert candle_mock.call_args_list[0][1]['indicators1'] == ['sma5', 'ema10']
+    assert candle_mock.call_args_list[0][1]['indicators2'] == ['macd']
+
+    assert log_has("End of plotting process. 2 plots generated", caplog)
+
+
+def test_start_plot_profit(mocker):
+    aup = mocker.patch("freqtrade.plot.plotting.plot_profit", MagicMock())
+    args = [
+        "--config", "config.json.example",
+        "plot-profit",
+        "--pairs", "ETH/BTC"
+    ]
+    start_plot_profit(get_args(args))
+
+    assert aup.call_count == 1
+    called_config = aup.call_args_list[0][0][0]
+    assert "pairs" in called_config
+    assert called_config['pairs'] == ["ETH/BTC"]
+
+
+def test_start_plot_profit_error(mocker):
+
+    args = [
+        "plot-profit",
+        "--pairs", "ETH/BTC"
+    ]
+    argsp = get_args(args)
+    # Make sure we use no config. Details: #2241
+    # not resetting config causes random failures if config.json exists
+    argsp["config"] = []
+    with pytest.raises(OperationalException):
+        start_plot_profit(argsp)
+
+
+def test_plot_profit(default_conf, mocker, testdatadir, caplog):
+    default_conf['trade_source'] = 'file'
+    default_conf["datadir"] = testdatadir
+    default_conf['exportfilename'] = str(testdatadir / "backtest-result_test.json")
+    default_conf['pairs'] = ["ETH/BTC", "LTC/BTC"]
+
+    profit_mock = MagicMock()
+    store_mock = MagicMock()
+    mocker.patch.multiple(
+        "freqtrade.plot.plotting",
+        generate_profit_graph=profit_mock,
+        store_plot_file=store_mock
+    )
+    plot_profit(default_conf)
+
+    # Plot-profit generates one combined plot
+    assert profit_mock.call_count == 1
+    assert store_mock.call_count == 1
+
+    assert profit_mock.call_args_list[0][0][0] == default_conf['pairs']
+    assert store_mock.call_args_list[0][1]['auto_open'] is True
diff --git a/freqtrade/tests/test_talib.py b/tests/test_talib.py
similarity index 100%
rename from freqtrade/tests/test_talib.py
rename to tests/test_talib.py
diff --git a/freqtrade/tests/test_timerange.py b/tests/test_timerange.py
similarity index 100%
rename from freqtrade/tests/test_timerange.py
rename to tests/test_timerange.py
diff --git a/freqtrade/tests/test_utils.py b/tests/test_utils.py
similarity index 66%
rename from freqtrade/tests/test_utils.py
rename to tests/test_utils.py
index d04e62b28..c99044610 100644
--- a/freqtrade/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -4,10 +4,11 @@ from unittest.mock import MagicMock, PropertyMock
 
 import pytest
 
+from freqtrade import OperationalException
 from freqtrade.state import RunMode
-from freqtrade.tests.conftest import get_args, log_has, patch_exchange
 from freqtrade.utils import (setup_utils_configuration, start_create_userdir,
                              start_download_data, start_list_exchanges)
+from tests.conftest import get_args, log_has, patch_exchange
 
 
 def test_setup_utils_configuration():
@@ -70,74 +71,8 @@ def test_create_datadir(caplog, mocker):
     assert len(caplog.record_tuples) == 0
 
 
-def test_download_data(mocker, markets, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
-    patch_exchange(mocker)
-    mocker.patch(
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    mocker.patch.object(Path, "unlink", MagicMock())
-
-    args = [
-        "download-data",
-        "--exchange", "binance",
-        "--pairs", "ETH/BTC", "XRP/BTC",
-        "--erase",
-    ]
-    start_download_data(get_args(args))
-
-    assert dl_mock.call_count == 4
-    assert dl_mock.call_args[1]['timerange'].starttype is None
-    assert dl_mock.call_args[1]['timerange'].stoptype is None
-    assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog)
-    assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
-
-
-def test_download_data_days(mocker, markets, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
-    patch_exchange(mocker)
-    mocker.patch(
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    mocker.patch.object(Path, "unlink", MagicMock())
-
-    args = [
-        "download-data",
-        "--exchange", "binance",
-        "--pairs", "ETH/BTC", "XRP/BTC",
-        "--days", "20",
-    ]
-
-    start_download_data(get_args(args))
-
-    assert dl_mock.call_count == 4
-    assert dl_mock.call_args[1]['timerange'].starttype == 'date'
-
-    assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
-
-
-def test_download_data_no_markets(mocker, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
-    patch_exchange(mocker)
-    mocker.patch(
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
-    )
-    args = [
-        "download-data",
-        "--exchange", "binance",
-        "--pairs", "ETH/BTC", "XRP/BTC",
-    ]
-    start_download_data(get_args(args))
-
-    assert dl_mock.call_count == 0
-    assert log_has("Skipping pair ETH/BTC...", caplog)
-    assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)
-
-
 def test_download_data_keyboardInterrupt(mocker, caplog, markets):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history',
+    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
                            MagicMock(side_effect=KeyboardInterrupt))
     patch_exchange(mocker)
     mocker.patch(
@@ -152,3 +87,60 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets):
         start_download_data(get_args(args))
 
     assert dl_mock.call_count == 1
+
+
+def test_download_data_no_markets(mocker, caplog):
+    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
+                           MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    args = [
+        "download-data",
+        "--exchange", "binance",
+        "--pairs", "ETH/BTC", "XRP/BTC",
+        "--days", "20"
+    ]
+    start_download_data(get_args(args))
+    assert dl_mock.call_args[1]['timerange'].starttype == "date"
+    assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)
+
+
+def test_download_data_no_exchange(mocker, caplog):
+    mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
+                 MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    args = [
+        "download-data",
+        ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    with pytest.raises(OperationalException,
+                       match=r"This command requires a configured exchange.*"):
+        start_download_data(pargs)
+
+
+def test_download_data_no_pairs(mocker, caplog):
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+
+    mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
+                 MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    args = [
+        "download-data",
+        "--exchange",
+        "binance",
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    with pytest.raises(OperationalException,
+                       match=r"Downloading data requires a list of pairs\..*"):
+        start_download_data(pargs)
diff --git a/freqtrade/tests/test_wallets.py b/tests/test_wallets.py
similarity index 97%
rename from freqtrade/tests/test_wallets.py
rename to tests/test_wallets.py
index 2c493cfc3..ae2810a2d 100644
--- a/freqtrade/tests/test_wallets.py
+++ b/tests/test_wallets.py
@@ -1,5 +1,5 @@
 # pragma pylint: disable=missing-docstring
-from freqtrade.tests.conftest import get_patched_freqtradebot
+from tests.conftest import get_patched_freqtradebot
 from unittest.mock import MagicMock
 
 
diff --git a/freqtrade/tests/testdata/ADA_BTC-1m.json b/tests/testdata/ADA_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/ADA_BTC-1m.json
rename to tests/testdata/ADA_BTC-1m.json
diff --git a/freqtrade/tests/testdata/ADA_BTC-5m.json b/tests/testdata/ADA_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/ADA_BTC-5m.json
rename to tests/testdata/ADA_BTC-5m.json
diff --git a/freqtrade/tests/testdata/DASH_BTC-1m.json b/tests/testdata/DASH_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/DASH_BTC-1m.json
rename to tests/testdata/DASH_BTC-1m.json
diff --git a/freqtrade/tests/testdata/DASH_BTC-5m.json b/tests/testdata/DASH_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/DASH_BTC-5m.json
rename to tests/testdata/DASH_BTC-5m.json
diff --git a/freqtrade/tests/testdata/ETC_BTC-1m.json b/tests/testdata/ETC_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/ETC_BTC-1m.json
rename to tests/testdata/ETC_BTC-1m.json
diff --git a/freqtrade/tests/testdata/ETC_BTC-5m.json b/tests/testdata/ETC_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/ETC_BTC-5m.json
rename to tests/testdata/ETC_BTC-5m.json
diff --git a/freqtrade/tests/testdata/ETH_BTC-1m.json b/tests/testdata/ETH_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/ETH_BTC-1m.json
rename to tests/testdata/ETH_BTC-1m.json
diff --git a/freqtrade/tests/testdata/ETH_BTC-5m.json b/tests/testdata/ETH_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/ETH_BTC-5m.json
rename to tests/testdata/ETH_BTC-5m.json
diff --git a/freqtrade/tests/testdata/LTC_BTC-1m.json b/tests/testdata/LTC_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/LTC_BTC-1m.json
rename to tests/testdata/LTC_BTC-1m.json
diff --git a/freqtrade/tests/testdata/LTC_BTC-5m.json b/tests/testdata/LTC_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/LTC_BTC-5m.json
rename to tests/testdata/LTC_BTC-5m.json
diff --git a/freqtrade/tests/testdata/NXT_BTC-1m.json b/tests/testdata/NXT_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/NXT_BTC-1m.json
rename to tests/testdata/NXT_BTC-1m.json
diff --git a/freqtrade/tests/testdata/NXT_BTC-5m.json b/tests/testdata/NXT_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/NXT_BTC-5m.json
rename to tests/testdata/NXT_BTC-5m.json
diff --git a/freqtrade/tests/testdata/POWR_BTC-1m.json b/tests/testdata/POWR_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/POWR_BTC-1m.json
rename to tests/testdata/POWR_BTC-1m.json
diff --git a/freqtrade/tests/testdata/POWR_BTC-5m.json b/tests/testdata/POWR_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/POWR_BTC-5m.json
rename to tests/testdata/POWR_BTC-5m.json
diff --git a/freqtrade/tests/testdata/UNITTEST_BTC-1m.json b/tests/testdata/UNITTEST_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/UNITTEST_BTC-1m.json
rename to tests/testdata/UNITTEST_BTC-1m.json
diff --git a/freqtrade/tests/testdata/UNITTEST_BTC-30m.json b/tests/testdata/UNITTEST_BTC-30m.json
similarity index 100%
rename from freqtrade/tests/testdata/UNITTEST_BTC-30m.json
rename to tests/testdata/UNITTEST_BTC-30m.json
diff --git a/freqtrade/tests/testdata/UNITTEST_BTC-5m.json b/tests/testdata/UNITTEST_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/UNITTEST_BTC-5m.json
rename to tests/testdata/UNITTEST_BTC-5m.json
diff --git a/freqtrade/tests/testdata/UNITTEST_BTC-8m.json b/tests/testdata/UNITTEST_BTC-8m.json
similarity index 100%
rename from freqtrade/tests/testdata/UNITTEST_BTC-8m.json
rename to tests/testdata/UNITTEST_BTC-8m.json
diff --git a/freqtrade/tests/testdata/UNITTEST_BTC-8m.json.gz b/tests/testdata/UNITTEST_BTC-8m.json.gz
similarity index 100%
rename from freqtrade/tests/testdata/UNITTEST_BTC-8m.json.gz
rename to tests/testdata/UNITTEST_BTC-8m.json.gz
diff --git a/freqtrade/tests/testdata/XLM_BTC-1m.json b/tests/testdata/XLM_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/XLM_BTC-1m.json
rename to tests/testdata/XLM_BTC-1m.json
diff --git a/freqtrade/tests/testdata/XLM_BTC-5m.json b/tests/testdata/XLM_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/XLM_BTC-5m.json
rename to tests/testdata/XLM_BTC-5m.json
diff --git a/freqtrade/tests/testdata/XMR_BTC-1m.json b/tests/testdata/XMR_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/XMR_BTC-1m.json
rename to tests/testdata/XMR_BTC-1m.json
diff --git a/freqtrade/tests/testdata/XMR_BTC-5m.json b/tests/testdata/XMR_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/XMR_BTC-5m.json
rename to tests/testdata/XMR_BTC-5m.json
diff --git a/freqtrade/tests/testdata/ZEC_BTC-1m.json b/tests/testdata/ZEC_BTC-1m.json
similarity index 100%
rename from freqtrade/tests/testdata/ZEC_BTC-1m.json
rename to tests/testdata/ZEC_BTC-1m.json
diff --git a/freqtrade/tests/testdata/ZEC_BTC-5m.json b/tests/testdata/ZEC_BTC-5m.json
similarity index 100%
rename from freqtrade/tests/testdata/ZEC_BTC-5m.json
rename to tests/testdata/ZEC_BTC-5m.json
diff --git a/freqtrade/tests/testdata/backtest-result_test.json b/tests/testdata/backtest-result_test.json
similarity index 100%
rename from freqtrade/tests/testdata/backtest-result_test.json
rename to tests/testdata/backtest-result_test.json
diff --git a/freqtrade/tests/testdata/pairs.json b/tests/testdata/pairs.json
similarity index 100%
rename from freqtrade/tests/testdata/pairs.json
rename to tests/testdata/pairs.json
diff --git a/user_data/notebooks/strategy_analysis_example.ipynb b/user_data/notebooks/strategy_analysis_example.ipynb
index 014f4ca90..89d71fe9d 100644
--- a/user_data/notebooks/strategy_analysis_example.ipynb
+++ b/user_data/notebooks/strategy_analysis_example.ipynb
@@ -52,7 +52,7 @@
     "# Define some constants\n",
     "ticker_interval = \"5m\"\n",
     "# Name of the strategy class\n",
-    "strategy_name = 'TestStrategy'\n",
+    "strategy_name = 'SampleStrategy'\n",
     "# Path to user data\n",
     "user_data_dir = 'user_data'\n",
     "# Location of the strategy\n",
diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/sample_strategy.py
similarity index 97%
rename from user_data/strategies/test_strategy.py
rename to user_data/strategies/sample_strategy.py
index d8ff790b2..0649c6f94 100644
--- a/user_data/strategies/test_strategy.py
+++ b/user_data/strategies/sample_strategy.py
@@ -11,10 +11,9 @@ import numpy  # noqa
 
 
 # This class is a sample. Feel free to customize it.
-class TestStrategy(IStrategy):
-    __test__ = False  # pytest expects to find tests here because of the name
+class SampleStrategy(IStrategy):
     """
-    This is a test strategy to inspire you.
+    This is a sample strategy to inspire you.
     More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md
 
     You can:
@@ -28,6 +27,9 @@ class TestStrategy(IStrategy):
     - the prototype for the methods: minimal_roi, stoploss, populate_indicators, populate_buy_trend,
     populate_sell_trend, hyperopt_space, buy_strategy_generator
     """
+    # Strategy intervace version - allow new iterations of the strategy interface.
+    # Check the documentation or the Sample strategy to get the latest version.
+    INTERFACE_VERSION = 2
 
     # Minimal ROI designed for the strategy.
     # This attribute will be overridden if the config file contains "minimal_roi"
@@ -256,14 +258,14 @@ class TestStrategy(IStrategy):
         # Retrieve best bid and best ask
         # ------------------------------------
         """
-        # first check if dataprovider is available 
+        # first check if dataprovider is available
         if self.dp:
             if self.dp.runmode in ('live', 'dry_run'):
                 ob = self.dp.orderbook(metadata['pair'], 1)
                 dataframe['best_bid'] = ob['bids'][0][0]
                 dataframe['best_ask'] = ob['asks'][0][0]
         """
-        
+
         return dataframe
 
     def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: