Merge pull request #2661 from freqtrade/wallet_dry

Introduce Dry-Run Wallet

docs/configuration.md

@@ -47,7 +47,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `ticker_interval` | The ticker interval to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy). <br> ***Datatype:*** *String*
 | `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency). <br> ***Datatype:*** *String*
 | `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode. <br>*Defaults to `true`.* <br> ***Datatype:*** *Boolean*
-| `dry_run_wallet` | Overrides the default amount of 999.9 stake currency units in the wallet used by the bot running in the Dry Run mode if you need it for any reason. <br> ***Datatype:*** *Float*
+| `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in the Dry Run mode.<br>*Defaults to `1000`.* <br> ***Datatype:*** *Float*
 | `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.*  <br> ***Datatype:*** *Boolean*
 | `minimal_roi` | **Required.** Set the threshold in percent the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy). <br> ***Datatype:*** *Dict*
 | `stoploss` |  **Required.** Value of the stoploss in percent used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).  <br> ***Datatype:*** *Float (as ratio)*
@@ -149,6 +149,9 @@ In this case a trade amount is calculated as:
 currency_balance / (max_open_trades - current_open_trades)
 ```
 
+!!! Note "When using Dry-Run Mode"
+    When using `"stake_amount" : "unlimited",` in combination with Dry-Run, the balance will be simulated starting with a stake of `dry_run_wallet` which will evolve over time. It is therefore important to set `dry_run_wallet` to a sensible value (like 0.05 or 0.01 for BTC and 1000 or 100 for USDT, for example), otherwise it may simulate trades with 100 BTC (or more) or 0.05 USDT (or less) at once - which may not correspond to your real available balance or is less than the exchange minimal limit for the order amount for the stake currency.
+
 ### Understand minimal_roi
 
 The `minimal_roi` configuration parameter is a JSON object where the key is a duration
@@ -501,8 +504,10 @@ creating trades on the exchange.
 }
 ```
 
-Once you will be happy with your bot performance running in the Dry-run mode,
-you can switch it to production mode.
+Once you will be happy with your bot performance running in the Dry-run mode, you can switch it to production mode.
+
+!!! Note
+    A simulated wallet is available during dry-run mode, and will assume a starting capital of `dry_run_wallet` (defaults to 1000).
 
 ## Switch to production mode
 
@@ -532,7 +537,7 @@ you run it in production mode.
 ```
 
 !!! Note
-    If you have an exchange API key yet, [see our tutorial](/pre-requisite).
+    If you have an exchange API key yet, [see our tutorial](installation.md#setup-your-exchange-account).
 
 You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange.
 

freqtrade/constants.py

@@ -18,7 +18,7 @@ REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']
 ORDERTYPE_POSSIBILITIES = ['limit', 'market']
 ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'PriceFilter']
-DRY_RUN_WALLET = 999.9
+DRY_RUN_WALLET = 1000
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons
 
 USERPATH_HYPEROPTS = 'hyperopts'
@@ -75,7 +75,7 @@ CONF_SCHEMA = {
         },
         'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT},
         'dry_run': {'type': 'boolean'},
-        'dry_run_wallet': {'type': 'number'},
+        'dry_run_wallet': {'type': 'number', 'default': DRY_RUN_WALLET},
         'process_only_new_candles': {'type': 'boolean'},
         'minimal_roi': {
             'type': 'object',
@@ -275,6 +275,7 @@ CONF_SCHEMA = {
         'stake_currency',
         'stake_amount',
         'dry_run',
+        'dry_run_wallet',
         'bid_strategy',
         'unfilledtimeout',
         'stoploss',

freqtrade/exchange/exchange.py

@@ -18,7 +18,7 @@ from ccxt.base.decimal_to_precision import ROUND_DOWN, ROUND_UP
 from pandas import DataFrame
 
 from freqtrade import (DependencyException, InvalidOrderException,
-                       OperationalException, TemporaryError, constants)
+                       OperationalException, TemporaryError)
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
 from freqtrade.misc import deep_merge_dicts
@@ -479,7 +479,7 @@ class Exchange:
     @retrier
     def get_balance(self, currency: str) -> float:
         if self._config['dry_run']:
-            return constants.DRY_RUN_WALLET
+            return self._config['dry_run_wallet']
 
         # ccxt exception is already handled by get_balances
         balances = self.get_balances()

freqtrade/freqtradebot.py

@@ -62,7 +62,11 @@ class FreqtradeBot:
 
         self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange
 
+        persistence.init(self.config.get('db_url', None),
+                         clean_open_orders=self.config.get('dry_run', False))
+
         self.wallets = Wallets(self.config, self.exchange)
+
         self.dataprovider = DataProvider(self.config, self.exchange)
 
         # Attach Dataprovider to Strategy baseclass
@@ -78,9 +82,6 @@ class FreqtradeBot:
 
         self.active_pair_whitelist = self._refresh_whitelist()
 
-        persistence.init(self.config.get('db_url', None),
-                         clean_open_orders=self.config.get('dry_run', False))
-
         # Set initial bot state from config
         initial_state = self.config.get('initial_state')
         self.state = State[initial_state.upper()] if initial_state else State.STOPPED

freqtrade/rpc/rpc.py

@@ -348,6 +348,7 @@ class RPC:
             'total': total,
             'symbol': symbol,
             'value': value,
+            'note': 'Simulated balances' if self._freqtrade.config.get('dry_run', False) else ''
         }
 
     def _rpc_start(self) -> Dict[str, str]:

freqtrade/rpc/telegram.py

@@ -331,7 +331,15 @@ class Telegram(RPC):
         try:
             result = self._rpc_balance(self._config['stake_currency'],
                                        self._config.get('fiat_display_currency', ''))
+
             output = ''
+            if self._config['dry_run']:
+                output += (
+                    f"*Warning:*Simulated balances in Dry Mode.\n"
+                    "This mode is still experimental!\n"
+                    "Starting capital: "
+                    f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n"
+                    )
             for currency in result['currencies']:
                 if currency['est_stake'] > 0.0001:
                     curr_output = "*{currency}:*\n" \

freqtrade/wallets.py

@@ -4,7 +4,7 @@
 import logging
 from typing import Dict, NamedTuple, Any
 from freqtrade.exchange import Exchange
-from freqtrade import constants
+from freqtrade.persistence import Trade
 
 logger = logging.getLogger(__name__)
 
@@ -23,14 +23,12 @@ class Wallets:
         self._config = config
         self._exchange = exchange
         self._wallets: Dict[str, Wallet] = {}
+        self.start_cap = config['dry_run_wallet']
 
         self.update()
 
     def get_free(self, currency) -> float:
 
-        if self._config['dry_run']:
-            return self._config.get('dry_run_wallet', constants.DRY_RUN_WALLET)
-
         balance = self._wallets.get(currency)
         if balance and balance.free:
             return balance.free
@@ -39,9 +37,6 @@ class Wallets:
 
     def get_used(self, currency) -> float:
 
-        if self._config['dry_run']:
-            return self._config.get('dry_run_wallet', constants.DRY_RUN_WALLET)
-
         balance = self._wallets.get(currency)
         if balance and balance.used:
             return balance.used
@@ -50,16 +45,42 @@ class Wallets:
 
     def get_total(self, currency) -> float:
 
-        if self._config['dry_run']:
-            return self._config.get('dry_run_wallet', constants.DRY_RUN_WALLET)
-
         balance = self._wallets.get(currency)
         if balance and balance.total:
             return balance.total
         else:
             return 0
 
-    def update(self) -> None:
+    def _update_dry(self) -> None:
+        """
+        Update from database in dry-run mode
+        - Apply apply profits of closed trades on top of stake amount
+        - Subtract currently tied up stake_amount in open trades
+        - update balances for currencies currently in trades
+        """
+        closed_trades = Trade.get_trades(Trade.is_open.is_(False)).all()
+        open_trades = Trade.get_trades(Trade.is_open.is_(True)).all()
+        tot_profit = sum([trade.calc_profit() for trade in closed_trades])
+        tot_in_trades = sum([trade.stake_amount for trade in open_trades])
+
+        current_stake = self.start_cap + tot_profit - tot_in_trades
+        self._wallets[self._config['stake_currency']] = Wallet(
+            self._config['stake_currency'],
+            current_stake,
+            0,
+            current_stake
+        )
+
+        for trade in open_trades:
+            curr = trade.pair.split('/')[0]
+            self._wallets[curr] = Wallet(
+                curr,
+                trade.amount,
+                0,
+                trade.amount
+            )
+
+    def _update_live(self) -> None:
 
         balances = self._exchange.get_balances()
 
@@ -71,6 +92,11 @@ class Wallets:
                 balances[currency].get('total', None)
             )
 
+    def update(self) -> None:
+        if self._config['dry_run']:
+            self._update_dry()
+        else:
+            self._update_live()
         logger.info('Wallets synced.')
 
     def get_all_balances(self) -> Dict[str, Any]:

tests/exchange/test_exchange.py

@@ -876,6 +876,7 @@ def test_sell_considers_time_in_force(default_conf, mocker, exchange_name):
 
 def test_get_balance_dry_run(default_conf, mocker):
     default_conf['dry_run'] = True
+    default_conf['dry_run_wallet'] = 999.9
 
     exchange = get_patched_exchange(mocker, default_conf)
     assert exchange.get_balance(currency='BTC') == 999.9

tests/rpc/test_rpc.py

@@ -398,7 +398,7 @@ def test_rpc_balance_handle(default_conf, mocker, tickers):
         get_valid_pair_combination=MagicMock(
             side_effect=lambda a, b: f"{b}/{a}" if a == "USDT" else f"{a}/{b}")
     )
-
+    default_conf['dry_run'] = False
     freqtradebot = get_patched_freqtradebot(mocker, default_conf)
     patch_get_signal(freqtradebot, (True, False))
     rpc = RPC(freqtradebot)

tests/rpc/test_rpc_apiserver.py

@@ -230,6 +230,7 @@ def test_api_stopbuy(botclient):
 def test_api_balance(botclient, mocker, rpc_balance):
     ftbot, client = botclient
 
+    ftbot.config['dry_run'] = False
     mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance)
     mocker.patch('freqtrade.exchange.Exchange.get_valid_pair_combination',
                  side_effect=lambda a, b: f"{a}/{b}")

tests/rpc/test_rpc_telegram.py

@@ -462,7 +462,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee,
 
 
 def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tickers) -> None:
-
+    default_conf['dry_run'] = False
     mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance)
     mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)
     mocker.patch('freqtrade.exchange.Exchange.get_valid_pair_combination',
@@ -494,6 +494,7 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tick
 
 
 def test_balance_handle_empty_response(default_conf, update, mocker) -> None:
+    default_conf['dry_run'] = False
     mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value={})
 
     msg_mock = MagicMock()
@@ -533,7 +534,8 @@ def test_balance_handle_empty_response_dry(default_conf, update, mocker) -> None
     telegram._balance(update=update, context=MagicMock())
     result = msg_mock.call_args_list[0][0][0]
     assert msg_mock.call_count == 1
-    assert "Running in Dry Run, balances are not available." in result
+    assert "*Warning:*Simulated balances in Dry Mode." in result
+    assert "Starting capital: `1000` BTC" in result
 
 
 def test_balance_handle_too_large_response(default_conf, update, mocker) -> None:

tests/test_configuration.py

@@ -8,9 +8,9 @@ from pathlib import Path
 from unittest.mock import MagicMock
 
 import pytest
-from jsonschema import Draft4Validator, ValidationError, validate
+from jsonschema import ValidationError
 
-from freqtrade import OperationalException, constants
+from freqtrade import OperationalException
 from freqtrade.configuration import (Arguments, Configuration, check_exchange,
                                      remove_credentials,
                                      validate_config_consistency)
@@ -718,7 +718,8 @@ def test_load_config_warn_forcebuy(default_conf, mocker, caplog) -> None:
 
 
 def test_validate_default_conf(default_conf) -> None:
-    validate(default_conf, constants.CONF_SCHEMA, Draft4Validator)
+    # Validate via our validator - we allow setting defaults!
+    validate_config_schema(default_conf)
 
 
 def test_validate_tsl(default_conf):

tests/test_freqtradebot.py

@@ -211,6 +211,7 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     patch_edge(mocker)
+    edge_conf['dry_run_wallet'] = 999.9
     freqtrade = FreqtradeBot(edge_conf)
 
     assert freqtrade._get_trade_stake_amount('NEO/BTC') == (999.9 * 0.5 * 0.01) / 0.20
@@ -1338,6 +1339,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog,
     patch_exchange(mocker)
     patch_edge(mocker)
     edge_conf['max_open_trades'] = float('inf')
+    edge_conf['dry_run_wallet'] = 999.9
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         get_ticker=MagicMock(return_value={
@@ -3483,3 +3485,32 @@ def test_process_i_am_alive(default_conf, mocker, caplog):
 
     ftbot.process()
     assert log_has_re(message, caplog)
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order):
+    default_conf['dry_run'] = True
+    # Initialize to 2 times stake amount
+    default_conf['dry_run_wallet'] = 0.002
+    default_conf['max_open_trades'] = 2
+    patch_exchange(mocker)
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        get_ticker=ticker,
+        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        get_fee=fee,
+    )
+
+    bot = get_patched_freqtradebot(mocker, default_conf)
+    patch_get_signal(bot)
+    assert bot.wallets.get_free('BTC') == 0.002
+
+    bot.create_trades()
+    trades = Trade.query.all()
+    assert len(trades) == 2
+
+    bot.config['max_open_trades'] = 3
+    with pytest.raises(
+            DependencyException,
+            match=r"Available balance \(0 BTC\) is lower than stake amount \(0.001 BTC\)"):
+        bot.create_trades()

tests/test_integration.py

@@ -71,6 +71,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee,
     )
     mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock)
     wallets_mock = mocker.patch("freqtrade.wallets.Wallets.update", MagicMock())
+    mocker.patch("freqtrade.wallets.Wallets.get_free", MagicMock(return_value=1))
 
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     freqtrade.strategy.order_types['stoploss_on_exchange'] = True

tests/test_wallets.py

@@ -1,7 +1,8 @@
 # pragma pylint: disable=missing-docstring
-from tests.conftest import get_patched_freqtradebot
 from unittest.mock import MagicMock
 
+from tests.conftest import get_patched_freqtradebot
+
 
 def test_sync_wallet_at_boot(mocker, default_conf):
     default_conf['dry_run'] = False