diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 2749d1281..debd5bc1b 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -11,34 +11,129 @@ The call sequence of the methods described here is covered under [bot execution !!! Tip Start off with a strategy template containing all available callback methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced` -## Storing information +## Storing information (Persistent) -Storing information can be accomplished by creating a new dictionary within the strategy class. +Freqtrade allows storing/retrieving user custom information associated with a specific trade in the database. -The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables. +Using a trade object, information can be stored using `trade.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object. + +For the data to be able to be stored within the database, freqtrade must serialized the data. This is done by converting the data to a JSON formatted string. +Freqtrade will attempt to reverse this action on retrieval, so from a strategy perspective, this should not be relevant. ```python +from freqtrade.persistence import Trade +from datetime import timedelta + class AwesomeStrategy(IStrategy): - # Create custom dictionary - custom_info = {} - def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - # Check if the entry already exists - if not metadata["pair"] in self.custom_info: - # Create empty entry for this pair - self.custom_info[metadata["pair"]] = {} + def bot_loop_start(self, **kwargs) -> None: + for trade in Trade.get_open_order_trades(): + fills = trade.select_filled_orders(trade.entry_side) + if trade.pair == 'ETH/USDT': + trade_entry_type = trade.get_custom_data(key='entry_type') + if trade_entry_type is None: + trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip' + elif fills > 1: + trade_entry_type = 'buy_up' + trade.set_custom_data(key='entry_type', value=trade_entry_type) + return super().bot_loop_start(**kwargs) - if "crosstime" in self.custom_info[metadata["pair"]]: - self.custom_info[metadata["pair"]]["crosstime"] += 1 - else: - self.custom_info[metadata["pair"]]["crosstime"] = 1 + def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str, + current_time: datetime, proposed_rate: float, current_order_rate: float, + entry_tag: Optional[str], side: str, **kwargs) -> float: + # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair. + if ( + pair == 'BTC/USDT' + and entry_tag == 'long_sma200' + and side == 'long' + and (current_time - timedelta(minutes=10)) > trade.open_date_utc + and order.filled == 0.0 + ): + dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) + current_candle = dataframe.iloc[-1].squeeze() + # store information about entry adjustment + existing_count = trade.get_custom_data('num_entry_adjustments', default=0) + if not existing_count: + existing_count = 1 + else: + existing_count += 1 + trade.set_custom_data(key='num_entry_adjustments', value=existing_count) + + # adjust order price + return current_candle['sma_200'] + + # default: maintain existing order + return current_order_rate + + def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs): + + entry_adjustment_count = trade.get_custom_data(key='num_entry_adjustments') + trade_entry_type = trade.get_custom_data(key='entry_type') + if entry_adjustment_count is None: + if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc): + return True, 'exit_1' + else + if entry_adjustment_count > 0 and if current_profit > 0.05: + return True, 'exit_2' + if trade_entry_type == 'breakout' and current_profit > 0.1: + return True, 'exit_3 + + return False, None ``` -!!! Warning - The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. +The above is a simple example - there are simpler ways to retrieve trade data like entry-adjustments. !!! Note - If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. + It is recommended that simple data types are used `[bool, int, float, str]` to ensure no issues when serializing the data that needs to be stored. + Storing big junks of data may lead to unintended side-effects, like a database becoming big (and as a consequence, also slow). + +!!! Warning "Non-serializable data" + If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data. + +??? Note "All attributes" + custom-data has the following accessors through the Trade object (assumed as `trade` below): + + * `trade.get_custom_data(key='something', default=0)` - Returns the actual value given in the type provided. + * `trade.get_custom_data_entry(key='something')` - Returns the entry - including metadata. The value is accessible via `.value` property. + * `trade.set_custom_data(key='something', value={'some': 'value'})` - set or update the corresponding key for this trade. Value must be serializable - and we recommend to keep the stored data relatively small. + + "value" can be any type (both in setting and receiving) - but must be json serializable. + +## Storing information (Non-Persistent) + +!!! Warning "Deprecated" + This method of storing information is deprecated and we do advise against using non-persistent storage. + Please use [Persistent Storage](#storing-information-persistent) instead. + + It's content has therefore been collapsed. + +??? Abstract "Storing information" + Storing information can be accomplished by creating a new dictionary within the strategy class. + + The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables. + + ```python + class AwesomeStrategy(IStrategy): + # Create custom dictionary + custom_info = {} + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # Check if the entry already exists + if not metadata["pair"] in self.custom_info: + # Create empty entry for this pair + self.custom_info[metadata["pair"]] = {} + + if "crosstime" in self.custom_info[metadata["pair"]]: + self.custom_info[metadata["pair"]]["crosstime"] += 1 + else: + self.custom_info[metadata["pair"]]["crosstime"] = 1 + ``` + + !!! Warning + The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. + + !!! Note + If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. ## Dataframe access diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index e4dc02c76..2709baf9a 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -181,6 +181,7 @@ official commands. You can ask at any moment for help with `/help`. | `/locks` | Show currently locked pairs. | `/unlock ` | Remove the lock for this pair (or for this lock id). | `/marketdir [long | short | even | none]` | Updates the user managed variable that represents the current market direction. If no direction is provided, the currently set direction will be displayed. +| `/list_custom_data [key]` | List custom_data for Trade ID & Key combination. If no Key is supplied it will list all key-value pairs found for that Trade ID. | **Modify Trade states** | | `/forceexit | /fx ` | Instantly exits the given trade (Ignoring `minimum_roi`). | `/forceexit all | /fx all` | Instantly exits all open trades (Ignoring `minimum_roi`). diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8d16122ea..b01bcf32e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -33,8 +33,8 @@ from freqtrade.optimize.optimize_reports import (generate_backtest_stats, genera show_backtest_results, store_backtest_analysis_results, store_backtest_stats) -from freqtrade.persistence import (LocalTrade, Order, PairLocks, Trade, disable_database_use, - enable_database_use) +from freqtrade.persistence import (CustomDataWrapper, LocalTrade, Order, PairLocks, Trade, + disable_database_use, enable_database_use) from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -337,6 +337,7 @@ class Backtesting: self.disable_database_use() PairLocks.reset_locks() Trade.reset_trades() + CustomDataWrapper.reset_custom_data() self.rejected_trades = 0 self.timedout_entry_orders = 0 self.timedout_exit_orders = 0 diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index 6205174a7..d5584c22c 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 +from freqtrade.persistence.custom_data import CustomDataWrapper from freqtrade.persistence.key_value_store import KeyStoreKeys, KeyValueStore from freqtrade.persistence.models import init_db from freqtrade.persistence.pairlock_middleware import PairLocks diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py new file mode 100644 index 000000000..81a9e7ad6 --- /dev/null +++ b/freqtrade/persistence/custom_data.py @@ -0,0 +1,174 @@ +import json +import logging +from datetime import datetime +from typing import Any, ClassVar, List, Optional, Sequence + +from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.persistence.base import ModelBase, SessionType +from freqtrade.util import dt_now + + +logger = logging.getLogger(__name__) + + +class _CustomData(ModelBase): + """ + CustomData database model + Keeps records of metadata as key/value store + for trades or global persistant values + One to many relationship with Trades: + - One trade can have many metadata entries + - One metadata entry can only be associated with one Trade + """ + __tablename__ = 'trade_custom_data' + __allow_unmapped__ = True + session: ClassVar[SessionType] + + # Uniqueness should be ensured over pair, order_id + # its likely that order_id is unique per Pair on some exchanges. + __table_args__ = (UniqueConstraint('ft_trade_id', 'cd_key', name="_trade_id_cd_key"),) + + id = mapped_column(Integer, primary_key=True) + ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True) + + trade = relationship("Trade", back_populates="custom_data") + + cd_key: Mapped[str] = mapped_column(String(255), nullable=False) + cd_type: Mapped[str] = mapped_column(String(25), nullable=False) + cd_value: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=dt_now) + updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + + # Empty container value - not persisted, but filled with cd_value on query + value: Any = None + + def __repr__(self): + create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT) + if self.created_at is not None else None) + update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT) + if self.updated_at is not None else None) + return (f'CustomData(id={self.id}, key={self.cd_key}, type={self.cd_type}, ' + + f'value={self.cd_value}, trade_id={self.ft_trade_id}, created={create_time}, ' + + f'updated={update_time})') + + @classmethod + def query_cd(cls, key: Optional[str] = None, + trade_id: Optional[int] = None) -> Sequence['_CustomData']: + """ + Get all CustomData, if trade_id is not specified + return will be for generic values not tied to a trade + :param trade_id: id of the Trade + """ + filters = [] + if trade_id is not None: + filters.append(_CustomData.ft_trade_id == trade_id) + if key is not None: + filters.append(_CustomData.cd_key.ilike(key)) + + return _CustomData.session.scalars(select(_CustomData).filter(*filters)).all() + + +class CustomDataWrapper: + """ + CustomData middleware class + Abstracts the database layer away so it becomes optional - which will be necessary to support + backtesting and hyperopt in the future. + """ + + use_db = True + custom_data: List[_CustomData] = [] + unserialized_types = ['bool', 'float', 'int', 'str'] + + @staticmethod + def _convert_custom_data(data: _CustomData) -> _CustomData: + if data.cd_type in CustomDataWrapper.unserialized_types: + data.value = data.cd_value + if data.cd_type == 'bool': + data.value = data.cd_value.lower() == 'true' + elif data.cd_type == 'int': + data.value = int(data.cd_value) + elif data.cd_type == 'float': + data.value = float(data.cd_value) + else: + data.value = json.loads(data.cd_value) + return data + + @staticmethod + def reset_custom_data() -> None: + """ + Resets all key-value pairs. Only active for backtesting mode. + """ + if not CustomDataWrapper.use_db: + CustomDataWrapper.custom_data = [] + + @staticmethod + def delete_custom_data(trade_id: int) -> None: + _CustomData.session.query(_CustomData).filter(_CustomData.ft_trade_id == trade_id).delete() + _CustomData.session.commit() + + @staticmethod + def get_custom_data(*, trade_id: int, key: Optional[str] = None) -> List[_CustomData]: + + if CustomDataWrapper.use_db: + filters = [ + _CustomData.ft_trade_id == trade_id, + ] + if key is not None: + filters.append(_CustomData.cd_key.ilike(key)) + filtered_custom_data = _CustomData.session.scalars(select(_CustomData).filter( + *filters)).all() + + else: + filtered_custom_data = [ + data_entry for data_entry in CustomDataWrapper.custom_data + if (data_entry.ft_trade_id == trade_id) + ] + if key is not None: + filtered_custom_data = [ + data_entry for data_entry in filtered_custom_data + if (data_entry.cd_key.casefold() == key.casefold()) + ] + return [CustomDataWrapper._convert_custom_data(d) for d in filtered_custom_data] + + @staticmethod + def set_custom_data(trade_id: int, key: str, value: Any) -> None: + + value_type = type(value).__name__ + + if value_type not in CustomDataWrapper.unserialized_types: + try: + value_db = json.dumps(value) + except TypeError as e: + logger.warning(f"could not serialize {key} value due to {e}") + return + else: + value_db = str(value) + + if trade_id is None: + trade_id = 0 + + custom_data = CustomDataWrapper.get_custom_data(trade_id=trade_id, key=key) + if custom_data: + data_entry = custom_data[0] + data_entry.cd_value = value_db + data_entry.updated_at = dt_now() + else: + data_entry = _CustomData( + ft_trade_id=trade_id, + cd_key=key, + cd_type=value_type, + cd_value=value_db, + created_at=dt_now(), + ) + data_entry.value = value + + if CustomDataWrapper.use_db and value_db is not None: + _CustomData.session.add(data_entry) + _CustomData.session.commit() + else: + if not custom_data: + CustomDataWrapper.custom_data.append(data_entry) + # Existing data will have updated interactively. diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index e561e727b..1a69b271c 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -13,6 +13,7 @@ from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException from freqtrade.persistence.base import ModelBase +from freqtrade.persistence.custom_data import _CustomData from freqtrade.persistence.key_value_store import _KeyValueStoreModel from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock @@ -78,6 +79,8 @@ def init_db(db_url: str) -> None: Order.session = Trade.session PairLock.session = Trade.session _KeyValueStoreModel.session = Trade.session + _CustomData.session = scoped_session(sessionmaker(bind=engine, autoflush=True), + scopefunc=get_request_or_thread_id) previous_tables = inspect(engine).get_table_names() ModelBase.metadata.create_all(engine) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 407affe72..defeb0e3f 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -23,6 +23,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi from freqtrade.leverage import interest from freqtrade.misc import safe_value_fallback from freqtrade.persistence.base import ModelBase, SessionType +from freqtrade.persistence.custom_data import CustomDataWrapper, _CustomData from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts, dt_ts_none @@ -1214,6 +1215,40 @@ class LocalTrade: or (o.ft_is_open is True and o.status is not None) ] + def set_custom_data(self, key: str, value: Any) -> None: + """ + Set custom data for this trade + :param key: key of the custom data + :param value: value of the custom data (must be JSON serializable) + """ + CustomDataWrapper.set_custom_data(trade_id=self.id, key=key, value=value) + + def get_custom_data(self, key: str, default: Any = None) -> Any: + """ + Get custom data for this trade + :param key: key of the custom data + """ + data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key) + if data: + return data[0].value + return default + + def get_custom_data_entry(self, key: str) -> Optional[_CustomData]: + """ + Get custom data for this trade + :param key: key of the custom data + """ + data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key) + if data: + return data[0] + return None + + def get_all_custom_data(self) -> List[_CustomData]: + """ + Get all custom data for this trade + """ + return CustomDataWrapper.get_custom_data(trade_id=self.id) + @property def nr_of_successful_entries(self) -> int: """ @@ -1469,6 +1504,9 @@ class Trade(ModelBase, LocalTrade): orders: Mapped[List[Order]] = relationship( "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin", innerjoin=True) # type: ignore + custom_data: Mapped[List[_CustomData]] = relationship( + "_CustomData", cascade="all, delete-orphan", + lazy="raise") exchange: Mapped[str] = mapped_column(String(25), nullable=False) # type: ignore pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True) # type: ignore @@ -1572,6 +1610,8 @@ class Trade(ModelBase, LocalTrade): for order in self.orders: Order.session.delete(order) + CustomDataWrapper.delete_custom_data(trade_id=self.id) + Trade.session.delete(self) Trade.commit() diff --git a/freqtrade/persistence/usedb_context.py b/freqtrade/persistence/usedb_context.py index 6fffd2fb5..732f0b0f8 100644 --- a/freqtrade/persistence/usedb_context.py +++ b/freqtrade/persistence/usedb_context.py @@ -1,4 +1,5 @@ +from freqtrade.persistence.custom_data import CustomDataWrapper from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import Trade @@ -11,6 +12,7 @@ def disable_database_use(timeframe: str) -> None: PairLocks.use_db = False PairLocks.timeframe = timeframe Trade.use_db = False + CustomDataWrapper.use_db = False def enable_database_use() -> None: @@ -20,6 +22,7 @@ def enable_database_use() -> None: PairLocks.use_db = True PairLocks.timeframe = '' Trade.use_db = True + CustomDataWrapper.use_db = True class FtNoDBContext: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 6e8447d29..47646923d 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -999,6 +999,32 @@ class RPC: 'cancel_order_count': c_count, } + def _rpc_list_custom_data(self, trade_id: int, key: Optional[str]) -> List[Dict[str, Any]]: + # Query for trade + trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first() + if trade is None: + return [] + # Query custom_data + custom_data = [] + if key: + data = trade.get_custom_data(key=key) + if data: + custom_data = [data] + else: + custom_data = trade.get_all_custom_data() + return [ + { + 'id': data_entry.id, + 'ft_trade_id': data_entry.ft_trade_id, + 'cd_key': data_entry.cd_key, + 'cd_type': data_entry.cd_type, + 'cd_value': data_entry.cd_value, + 'created_at': data_entry.created_at, + 'updated_at': data_entry.updated_at + } + for data_entry in custom_data + ] + def _rpc_performance(self) -> List[Dict[str, Any]]: """ Handler for performance. diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 904b1fdbc..f7e7362ef 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -33,7 +33,7 @@ from freqtrade.misc import chunks, plural from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException, RPCHandler from freqtrade.rpc.rpc_types import RPCEntryMsg, RPCExitMsg, RPCOrderMsg, RPCSendMsg -from freqtrade.util import dt_humanize, fmt_coin, round_value +from freqtrade.util import dt_humanize, fmt_coin, format_date, round_value MAX_MESSAGE_LENGTH = MessageLimit.MAX_TEXT_LENGTH @@ -243,6 +243,7 @@ class Telegram(RPCHandler): CommandHandler('version', self._version), CommandHandler('marketdir', self._changemarketdir), CommandHandler('order', self._order), + CommandHandler('list_custom_data', self._list_custom_data), ] callbacks = [ CallbackQueryHandler(self._status_table, pattern='update_status_table'), @@ -1667,6 +1668,8 @@ class Telegram(RPCHandler): "*/marketdir [long | short | even | none]:* `Updates the user managed variable " "that represents the current market direction. If no direction is provided `" "`the currently set market direction will be output.` \n" + "*/list_custom_data :* `List custom_data for Trade ID & Key combo.`\n" + "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`" "_Statistics_\n" "------------\n" @@ -1689,7 +1692,7 @@ class Telegram(RPCHandler): "*/stats:* `Shows Wins / losses by Sell reason as well as " "Avg. holding durations for buys and sells.`\n" "*/help:* `This help message`\n" - "*/version:* `Show version`" + "*/version:* `Show version`\n" ) await self._send_msg(message, parse_mode=ParseMode.MARKDOWN) @@ -1766,6 +1769,53 @@ class Telegram(RPCHandler): f"*Current state:* `{val['state']}`" ) + @authorized_only + async def _list_custom_data(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /list_custom_data . + List custom_data for specified trade (and key if supplied). + :param bot: telegram bot + :param update: message update + :return: None + """ + try: + if not context.args or len(context.args) == 0: + raise RPCException("Trade-id not set.") + trade_id = int(context.args[0]) + key = None if len(context.args) < 2 else str(context.args[1]) + + results = self._rpc._rpc_list_custom_data(trade_id, key) + messages = [] + if len(results) > 0: + messages.append( + 'Found custom-data entr' + ('ies: ' if len(results) > 1 else 'y: ') + ) + for result in results: + lines = [ + f"*Key:* `{result['cd_key']}`", + f"*ID:* `{result['id']}`", + f"*Trade ID:* `{result['ft_trade_id']}`", + f"*Type:* `{result['cd_type']}`", + f"*Value:* `{result['cd_value']}`", + f"*Create Date:* `{format_date(result['created_at'])}`", + f"*Update Date:* `{format_date(result['updated_at'])}`" + ] + # Filter empty lines using list-comprehension + messages.append("\n".join([line for line in lines if line])) + for msg in messages: + if len(msg) > MAX_MESSAGE_LENGTH: + msg = "Message dropped because length exceeds " + msg += f"maximum allowed characters: {MAX_MESSAGE_LENGTH}" + logger.warning(msg) + await self._send_msg(msg) + else: + message = f"Didn't find any custom-data entries for Trade ID: `{trade_id}`" + message += f" and Key: `{key}`." if key is not None else "" + await self._send_msg(message) + + except RPCException as e: + await self._send_msg(str(e)) + async def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "", reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None: if reload_able: diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 0e0e70ee8..18f28da2b 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -2099,6 +2099,7 @@ def test_Trade_object_idem(): 'get_mix_tag_performance', 'get_trading_volume', 'validate_string_len', + 'custom_data' ) EXCLUDES2 = ('trades', 'trades_open', 'bt_trades_open_pp', 'bt_open_open_trade_count', 'total_profit', 'from_json',) diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py new file mode 100644 index 000000000..15241aa93 --- /dev/null +++ b/tests/persistence/test_trade_custom_data.py @@ -0,0 +1,160 @@ +from copy import deepcopy +from unittest.mock import MagicMock + +import pytest + +from freqtrade.data.history.history_utils import get_timerange +from freqtrade.optimize.backtesting import Backtesting +from freqtrade.persistence import Trade, disable_database_use, enable_database_use +from freqtrade.persistence.custom_data import CustomDataWrapper +from tests.conftest import (EXMS, create_mock_trades_usdt, generate_test_data, + get_patched_freqtradebot, patch_exchange) + + +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize("use_db", [True, False]) +def test_trade_custom_data(fee, use_db): + if not use_db: + disable_database_use('5m') + Trade.reset_trades() + CustomDataWrapper.reset_custom_data() + + create_mock_trades_usdt(fee, use_db=use_db) + + trade1 = Trade.get_trades_proxy()[0] + if not use_db: + trade1.id = 1 + + assert trade1.get_all_custom_data() == [] + trade1.set_custom_data('test_str', 'test_value') + trade1.set_custom_data('test_int', 1) + trade1.set_custom_data('test_float', 1.55) + trade1.set_custom_data('test_bool', True) + trade1.set_custom_data('test_dict', {'test': 'dict'}) + + assert len(trade1.get_all_custom_data()) == 5 + assert trade1.get_custom_data('test_str') == 'test_value' + trade1.set_custom_data('test_str', 'test_value_updated') + assert trade1.get_custom_data('test_str') == 'test_value_updated' + + assert trade1.get_custom_data('test_int') == 1 + assert isinstance(trade1.get_custom_data('test_int'), int) + + assert trade1.get_custom_data('test_float') == 1.55 + assert isinstance(trade1.get_custom_data('test_float'), float) + + assert trade1.get_custom_data('test_bool') is True + assert isinstance(trade1.get_custom_data('test_bool'), bool) + + assert trade1.get_custom_data('test_dict') == {'test': 'dict'} + assert isinstance(trade1.get_custom_data('test_dict'), dict) + if not use_db: + enable_database_use() + + +def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee): + + mocker.patch(f'{EXMS}.get_rate', return_value=0.50) + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=None) + default_conf_usdt["minimal_roi"] = {"0": 100} + + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + create_mock_trades_usdt(fee) + + trade1 = Trade.get_trades_proxy(pair='ADA/USDT')[0] + trade1.set_custom_data('test_str', 'test_value') + trade1.set_custom_data('test_int', 1) + + def custom_exit(pair, trade, **kwargs): + + if pair == 'ADA/USDT': + custom_val = trade.get_custom_data('test_str') + custom_val_i = trade.get_custom_data('test_int') + + return f"{custom_val}_{custom_val_i}" + + freqtrade.strategy.custom_exit = custom_exit + ff_spy = mocker.spy(freqtrade.strategy, 'custom_exit') + trades = Trade.get_open_trades() + freqtrade.exit_positions(trades) + Trade.commit() + + trade_after = Trade.get_trades_proxy(pair='ADA/USDT')[0] + assert trade_after.get_custom_data('test_str') == 'test_value' + assert trade_after.get_custom_data('test_int') == 1 + # 2 open pairs eligible for exit + assert ff_spy.call_count == 2 + + assert trade_after.exit_reason == 'test_value_1' + + +def test_trade_custom_data_strategy_backtest_compat(mocker, default_conf_usdt, fee): + + mocker.patch(f'{EXMS}.get_fee', fee) + mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=10) + mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float('inf')) + mocker.patch(f"{EXMS}.get_max_leverage", return_value=10) + mocker.patch(f"{EXMS}.get_maintenance_ratio_and_amt", return_value=(0.1, 0.1)) + mocker.patch('freqtrade.optimize.backtesting.Backtesting._run_funding_fees') + + patch_exchange(mocker) + default_conf_usdt.update({ + "stake_amount": 100.0, + "max_open_trades": 2, + "dry_run_wallet": 1000.0, + "strategy": "StrategyTestV3", + "trading_mode": "futures", + "margin_mode": "isolated", + "stoploss": -2, + "minimal_roi": {"0": 100}, + }) + default_conf_usdt['pairlists'] = [{'method': 'StaticPairList', 'allow_inactive': True}] + backtesting = Backtesting(default_conf_usdt) + + df = generate_test_data(default_conf_usdt['timeframe'], 100, '2022-01-01 00:00:00+00:00') + + pair_exp = 'XRP/USDT:USDT' + + def custom_exit(pair, trade, **kwargs): + custom_val = trade.get_custom_data('test_str') + custom_val_i = trade.get_custom_data('test_int', 0) + + if pair == pair_exp: + trade.set_custom_data('test_str', 'test_value') + trade.set_custom_data('test_int', custom_val_i + 1) + + if custom_val_i >= 2: + return f"{custom_val}_{custom_val_i}" + + backtesting._set_strategy(backtesting.strategylist[0]) + processed = backtesting.strategy.advise_all_indicators({ + pair_exp: df, + 'BTC/USDT:USDT': df, + }) + + def fun(dataframe, *args, **kwargs): + dataframe.loc[dataframe.index == 50, 'enter_long'] = 1 + return dataframe + + backtesting.strategy.advise_entry = fun + backtesting.strategy.leverage = MagicMock(return_value=1) + backtesting.strategy.custom_exit = custom_exit + ff_spy = mocker.spy(backtesting.strategy, 'custom_exit') + + min_date, max_date = get_timerange(processed) + + result = backtesting.backtest( + processed=deepcopy(processed), + start_date=min_date, + end_date=max_date, + ) + results = result['results'] + assert not results.empty + assert len(results) == 2 + assert results['pair'][0] == pair_exp + assert results['pair'][1] == 'BTC/USDT:USDT' + assert results['exit_reason'][0] == 'test_value_2' + assert results['exit_reason'][1] == 'exit_signal' + + assert ff_spy.call_count == 7 + Backtesting.cleanup() diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 7b1347fd6..3bd372b19 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -150,7 +150,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: "['stopbuy', 'stopentry'], ['whitelist'], ['blacklist'], " "['bl_delete', 'blacklist_delete'], " "['logs'], ['edge'], ['health'], ['help'], ['version'], ['marketdir'], " - "['order']]") + "['order'], ['list_custom_data']]") assert log_has(message_str, caplog) @@ -2657,3 +2657,49 @@ async def test_change_market_direction(default_conf, mocker, update) -> None: context.args = ["invalid"] await telegram._changemarketdir(update, context) assert telegram._rpc._freqtrade.strategy.market_direction == MarketDirection.LONG + + +async def test_telegram_list_custom_data(default_conf_usdt, update, ticker, fee, mocker) -> None: + + mocker.patch.multiple( + EXMS, + fetch_ticker=ticker, + get_fee=fee, + ) + telegram, _freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) + + # Create some test data + create_mock_trades_usdt(fee) + # No trade id + context = MagicMock() + await telegram._list_custom_data(update=update, context=context) + assert msg_mock.call_count == 1 + assert 'Trade-id not set.' in msg_mock.call_args_list[0][0][0] + msg_mock.reset_mock() + + # + context.args = ['1'] + await telegram._list_custom_data(update=update, context=context) + assert msg_mock.call_count == 1 + assert ( + "Didn't find any custom-data entries for Trade ID: `1`" in msg_mock.call_args_list[0][0][0] + ) + msg_mock.reset_mock() + + # Add some custom data + trade1 = Trade.get_trades_proxy()[0] + trade1.set_custom_data('test_int', 1) + trade1.set_custom_data('test_dict', {'test': 'dict'}) + Trade.commit() + context.args = [f"{trade1.id}"] + await telegram._list_custom_data(update=update, context=context) + assert msg_mock.call_count == 3 + assert "Found custom-data entries: " in msg_mock.call_args_list[0][0][0] + assert ( + "*Key:* `test_int`\n*ID:* `1`\n*Trade ID:* `1`\n*Type:* `int`\n" + "*Value:* `1`\n*Create Date:*") in msg_mock.call_args_list[1][0][0] + assert ( + '*Key:* `test_dict`\n*ID:* `2`\n*Trade ID:* `1`\n*Type:* `dict`\n' + '*Value:* `{"test": "dict"}`\n*Create Date:* `') in msg_mock.call_args_list[2][0][0] + + msg_mock.reset_mock()