Merge pull request #6908 from eSeR1805/feature_keyval_storage

Persistent storage of user-custom information

docs/strategy-advanced.md

@@ -11,34 +11,129 @@ The call sequence of the methods described here is covered under [bot execution
 !!! Tip
     Start off with a strategy template containing all available callback methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced`
 
-## Storing information
+## Storing information (Persistent)
 
-Storing information can be accomplished by creating a new dictionary within the strategy class.
+Freqtrade allows storing/retrieving user custom information associated with a specific trade in the database.
 
-The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables.
+Using a trade object, information can be stored using `trade.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object.
+
+For the data to be able to be stored within the database, freqtrade must serialized the data. This is done by converting the data to a JSON formatted string.
+Freqtrade will attempt to reverse this action on retrieval, so from a strategy perspective, this should not be relevant.
 
 ```python
+from freqtrade.persistence import Trade
+from datetime import timedelta
+
 class AwesomeStrategy(IStrategy):
-    # Create custom dictionary
-    custom_info = {}
 
-    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+    def bot_loop_start(self, **kwargs) -> None:
-        # Check if the entry already exists
+        for trade in Trade.get_open_order_trades():
-        if not metadata["pair"] in self.custom_info:
+            fills = trade.select_filled_orders(trade.entry_side)
-            # Create empty entry for this pair
+            if trade.pair == 'ETH/USDT':
-            self.custom_info[metadata["pair"]] = {}
+                trade_entry_type = trade.get_custom_data(key='entry_type')
+                if trade_entry_type is None:
+                    trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip'
+                elif fills > 1:
+                    trade_entry_type = 'buy_up'
+                trade.set_custom_data(key='entry_type', value=trade_entry_type)
+        return super().bot_loop_start(**kwargs)
 
-        if "crosstime" in self.custom_info[metadata["pair"]]:
+    def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str,
-            self.custom_info[metadata["pair"]]["crosstime"] += 1
+                           current_time: datetime, proposed_rate: float, current_order_rate: float,
-        else:
+                           entry_tag: Optional[str], side: str, **kwargs) -> float:
-            self.custom_info[metadata["pair"]]["crosstime"] = 1
+        # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair.
+        if (
+            pair == 'BTC/USDT' 
+            and entry_tag == 'long_sma200' 
+            and side == 'long' 
+            and (current_time - timedelta(minutes=10)) > trade.open_date_utc 
+            and order.filled == 0.0
+        ):
+            dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe)
+            current_candle = dataframe.iloc[-1].squeeze()
+            # store information about entry adjustment
+            existing_count = trade.get_custom_data('num_entry_adjustments', default=0)
+            if not existing_count:
+                existing_count = 1
+            else:
+                existing_count += 1
+            trade.set_custom_data(key='num_entry_adjustments', value=existing_count)
+
+            # adjust order price
+            return current_candle['sma_200']
+
+        # default: maintain existing order
+        return current_order_rate
+
+    def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs):
+
+        entry_adjustment_count = trade.get_custom_data(key='num_entry_adjustments')
+        trade_entry_type = trade.get_custom_data(key='entry_type')
+        if entry_adjustment_count is None:
+            if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc):
+                return True, 'exit_1'
+        else
+            if entry_adjustment_count > 0 and if current_profit > 0.05:
+                return True, 'exit_2'
+            if trade_entry_type == 'breakout' and current_profit > 0.1:
+                return True, 'exit_3
+
+        return False, None
 ```
 
-!!! Warning
+The above is a simple example - there are simpler ways to retrieve trade data like entry-adjustments.
-    The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
 
 !!! Note
-    If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
+    It is recommended that simple data types are used `[bool, int, float, str]` to ensure no issues when serializing the data that needs to be stored.
+    Storing big junks of data may lead to unintended side-effects, like a database becoming big (and as a consequence, also slow).
+
+!!! Warning "Non-serializable data"
+    If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data.
+
+??? Note "All attributes"
+    custom-data has the following accessors through the Trade object (assumed as `trade` below):
+
+    * `trade.get_custom_data(key='something', default=0)` - Returns the actual value given in the type provided.
+    * `trade.get_custom_data_entry(key='something')` - Returns the entry - including metadata. The value is accessible via `.value` property.
+    * `trade.set_custom_data(key='something', value={'some': 'value'})` - set or update the corresponding key for this trade. Value must be serializable - and we recommend to keep the stored data relatively small.
+
+    "value" can be any type (both in setting and receiving) - but must be json serializable.
+
+## Storing information (Non-Persistent)
+
+!!! Warning "Deprecated"
+    This method of storing information is deprecated and we do advise against using non-persistent storage.  
+    Please use [Persistent Storage](#storing-information-persistent) instead.
+
+    It's content has therefore been collapsed.
+
+??? Abstract "Storing information"
+    Storing information can be accomplished by creating a new dictionary within the strategy class.
+
+    The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables.
+
+    ```python
+    class AwesomeStrategy(IStrategy):
+        # Create custom dictionary
+        custom_info = {}
+
+        def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+            # Check if the entry already exists
+            if not metadata["pair"] in self.custom_info:
+                # Create empty entry for this pair
+                self.custom_info[metadata["pair"]] = {}
+
+            if "crosstime" in self.custom_info[metadata["pair"]]:
+                self.custom_info[metadata["pair"]]["crosstime"] += 1
+            else:
+                self.custom_info[metadata["pair"]]["crosstime"] = 1
+    ```
+
+    !!! Warning
+        The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
+
+    !!! Note
+        If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
 
 ## Dataframe access
 

docs/telegram-usage.md

@@ -181,6 +181,7 @@ official commands. You can ask at any moment for help with `/help`.
 | `/locks` | Show currently locked pairs.
 | `/unlock <pair or lock_id>` | Remove the lock for this pair (or for this lock id).
 | `/marketdir [long | short | even | none]` | Updates the user managed variable that represents the current market direction. If no direction is provided, the currently set direction will be displayed.
+| `/list_custom_data <trade_id> [key]` | List custom_data for Trade ID & Key combination. If no Key is supplied it will list all key-value pairs found for that Trade ID.
 | **Modify Trade states** |
 | `/forceexit <trade_id> | /fx <tradeid>` | Instantly exits the given trade  (Ignoring `minimum_roi`).
 | `/forceexit all | /fx all` | Instantly exits all open trades (Ignoring `minimum_roi`).

freqtrade/optimize/backtesting.py

@@ -33,8 +33,8 @@ from freqtrade.optimize.optimize_reports import (generate_backtest_stats, genera
                                                  show_backtest_results,
                                                  store_backtest_analysis_results,
                                                  store_backtest_stats)
-from freqtrade.persistence import (LocalTrade, Order, PairLocks, Trade, disable_database_use,
+from freqtrade.persistence import (CustomDataWrapper, LocalTrade, Order, PairLocks, Trade,
-                                   enable_database_use)
+                                   disable_database_use, enable_database_use)
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -337,6 +337,7 @@ class Backtesting:
         self.disable_database_use()
         PairLocks.reset_locks()
         Trade.reset_trades()
+        CustomDataWrapper.reset_custom_data()
         self.rejected_trades = 0
         self.timedout_entry_orders = 0
         self.timedout_exit_orders = 0

freqtrade/persistence/__init__.py

@@ -1,5 +1,6 @@
 # flake8: noqa: F401
 
+from freqtrade.persistence.custom_data import CustomDataWrapper
 from freqtrade.persistence.key_value_store import KeyStoreKeys, KeyValueStore
 from freqtrade.persistence.models import init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks

freqtrade/persistence/custom_data.py

@@ -0,0 +1,174 @@
+import json
+import logging
+from datetime import datetime
+from typing import Any, ClassVar, List, Optional, Sequence
+
+from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.persistence.base import ModelBase, SessionType
+from freqtrade.util import dt_now
+
+
+logger = logging.getLogger(__name__)
+
+
+class _CustomData(ModelBase):
+    """
+    CustomData database model
+    Keeps records of metadata as key/value store
+    for trades or global persistant values
+    One to many relationship with Trades:
+      - One trade can have many metadata entries
+      - One metadata entry can only be associated with one Trade
+    """
+    __tablename__ = 'trade_custom_data'
+    __allow_unmapped__ = True
+    session: ClassVar[SessionType]
+
+    # Uniqueness should be ensured over pair, order_id
+    # its likely that order_id is unique per Pair on some exchanges.
+    __table_args__ = (UniqueConstraint('ft_trade_id', 'cd_key', name="_trade_id_cd_key"),)
+
+    id = mapped_column(Integer, primary_key=True)
+    ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True)
+
+    trade = relationship("Trade", back_populates="custom_data")
+
+    cd_key: Mapped[str] = mapped_column(String(255), nullable=False)
+    cd_type: Mapped[str] = mapped_column(String(25), nullable=False)
+    cd_value: Mapped[str] = mapped_column(Text, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=dt_now)
+    updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+
+    # Empty container value - not persisted, but filled with cd_value on query
+    value: Any = None
+
+    def __repr__(self):
+        create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT)
+                       if self.created_at is not None else None)
+        update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT)
+                       if self.updated_at is not None else None)
+        return (f'CustomData(id={self.id}, key={self.cd_key}, type={self.cd_type}, ' +
+                f'value={self.cd_value}, trade_id={self.ft_trade_id}, created={create_time}, ' +
+                f'updated={update_time})')
+
+    @classmethod
+    def query_cd(cls, key: Optional[str] = None,
+                 trade_id: Optional[int] = None) -> Sequence['_CustomData']:
+        """
+        Get all CustomData, if trade_id is not specified
+        return will be for generic values not tied to a trade
+        :param trade_id: id of the Trade
+        """
+        filters = []
+        if trade_id is not None:
+            filters.append(_CustomData.ft_trade_id == trade_id)
+        if key is not None:
+            filters.append(_CustomData.cd_key.ilike(key))
+
+        return _CustomData.session.scalars(select(_CustomData).filter(*filters)).all()
+
+
+class CustomDataWrapper:
+    """
+    CustomData middleware class
+    Abstracts the database layer away so it becomes optional - which will be necessary to support
+    backtesting and hyperopt in the future.
+    """
+
+    use_db = True
+    custom_data: List[_CustomData] = []
+    unserialized_types = ['bool', 'float', 'int', 'str']
+
+    @staticmethod
+    def _convert_custom_data(data: _CustomData) -> _CustomData:
+        if data.cd_type in CustomDataWrapper.unserialized_types:
+            data.value = data.cd_value
+            if data.cd_type == 'bool':
+                data.value = data.cd_value.lower() == 'true'
+            elif data.cd_type == 'int':
+                data.value = int(data.cd_value)
+            elif data.cd_type == 'float':
+                data.value = float(data.cd_value)
+        else:
+            data.value = json.loads(data.cd_value)
+        return data
+
+    @staticmethod
+    def reset_custom_data() -> None:
+        """
+        Resets all key-value pairs. Only active for backtesting mode.
+        """
+        if not CustomDataWrapper.use_db:
+            CustomDataWrapper.custom_data = []
+
+    @staticmethod
+    def delete_custom_data(trade_id: int) -> None:
+        _CustomData.session.query(_CustomData).filter(_CustomData.ft_trade_id == trade_id).delete()
+        _CustomData.session.commit()
+
+    @staticmethod
+    def get_custom_data(*, trade_id: int, key: Optional[str] = None) -> List[_CustomData]:
+
+        if CustomDataWrapper.use_db:
+            filters = [
+                _CustomData.ft_trade_id == trade_id,
+            ]
+            if key is not None:
+                filters.append(_CustomData.cd_key.ilike(key))
+            filtered_custom_data = _CustomData.session.scalars(select(_CustomData).filter(
+                *filters)).all()
+
+        else:
+            filtered_custom_data = [
+                data_entry for data_entry in CustomDataWrapper.custom_data
+                if (data_entry.ft_trade_id == trade_id)
+            ]
+            if key is not None:
+                filtered_custom_data = [
+                    data_entry for data_entry in filtered_custom_data
+                    if (data_entry.cd_key.casefold() == key.casefold())
+                ]
+        return [CustomDataWrapper._convert_custom_data(d) for d in filtered_custom_data]
+
+    @staticmethod
+    def set_custom_data(trade_id: int, key: str, value: Any) -> None:
+
+        value_type = type(value).__name__
+
+        if value_type not in CustomDataWrapper.unserialized_types:
+            try:
+                value_db = json.dumps(value)
+            except TypeError as e:
+                logger.warning(f"could not serialize {key} value due to {e}")
+                return
+        else:
+            value_db = str(value)
+
+        if trade_id is None:
+            trade_id = 0
+
+        custom_data = CustomDataWrapper.get_custom_data(trade_id=trade_id, key=key)
+        if custom_data:
+            data_entry = custom_data[0]
+            data_entry.cd_value = value_db
+            data_entry.updated_at = dt_now()
+        else:
+            data_entry = _CustomData(
+                ft_trade_id=trade_id,
+                cd_key=key,
+                cd_type=value_type,
+                cd_value=value_db,
+                created_at=dt_now(),
+            )
+        data_entry.value = value
+
+        if CustomDataWrapper.use_db and value_db is not None:
+            _CustomData.session.add(data_entry)
+            _CustomData.session.commit()
+        else:
+            if not custom_data:
+                CustomDataWrapper.custom_data.append(data_entry)
+            # Existing data will have updated interactively.

freqtrade/persistence/models.py

@@ -13,6 +13,7 @@ from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.base import ModelBase
+from freqtrade.persistence.custom_data import _CustomData
 from freqtrade.persistence.key_value_store import _KeyValueStoreModel
 from freqtrade.persistence.migrations import check_migrate
 from freqtrade.persistence.pairlock import PairLock
@@ -78,6 +79,8 @@ def init_db(db_url: str) -> None:
     Order.session = Trade.session
     PairLock.session = Trade.session
     _KeyValueStoreModel.session = Trade.session
+    _CustomData.session = scoped_session(sessionmaker(bind=engine, autoflush=True),
+                                         scopefunc=get_request_or_thread_id)
 
     previous_tables = inspect(engine).get_table_names()
     ModelBase.metadata.create_all(engine)

freqtrade/persistence/trade_model.py

@@ -23,6 +23,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi
 from freqtrade.leverage import interest
 from freqtrade.misc import safe_value_fallback
 from freqtrade.persistence.base import ModelBase, SessionType
+from freqtrade.persistence.custom_data import CustomDataWrapper, _CustomData
 from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts, dt_ts_none
 
 
@@ -1214,6 +1215,40 @@ class LocalTrade:
                 or (o.ft_is_open is True and o.status is not None)
                 ]
 
+    def set_custom_data(self, key: str, value: Any) -> None:
+        """
+        Set custom data for this trade
+        :param key: key of the custom data
+        :param value: value of the custom data (must be JSON serializable)
+        """
+        CustomDataWrapper.set_custom_data(trade_id=self.id, key=key, value=value)
+
+    def get_custom_data(self, key: str, default: Any = None) -> Any:
+        """
+        Get custom data for this trade
+        :param key: key of the custom data
+        """
+        data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key)
+        if data:
+            return data[0].value
+        return default
+
+    def get_custom_data_entry(self, key: str) -> Optional[_CustomData]:
+        """
+        Get custom data for this trade
+        :param key: key of the custom data
+        """
+        data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key)
+        if data:
+            return data[0]
+        return None
+
+    def get_all_custom_data(self) -> List[_CustomData]:
+        """
+        Get all custom data for this trade
+        """
+        return CustomDataWrapper.get_custom_data(trade_id=self.id)
+
     @property
     def nr_of_successful_entries(self) -> int:
         """
@@ -1469,6 +1504,9 @@ class Trade(ModelBase, LocalTrade):
     orders: Mapped[List[Order]] = relationship(
         "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin",
         innerjoin=True)  # type: ignore
+    custom_data: Mapped[List[_CustomData]] = relationship(
+        "_CustomData", cascade="all, delete-orphan",
+        lazy="raise")
 
     exchange: Mapped[str] = mapped_column(String(25), nullable=False)  # type: ignore
     pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True)  # type: ignore
@@ -1572,6 +1610,8 @@ class Trade(ModelBase, LocalTrade):
         for order in self.orders:
             Order.session.delete(order)
 
+        CustomDataWrapper.delete_custom_data(trade_id=self.id)
+
         Trade.session.delete(self)
         Trade.commit()
 

freqtrade/persistence/usedb_context.py

@@ -1,4 +1,5 @@
 
+from freqtrade.persistence.custom_data import CustomDataWrapper
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.persistence.trade_model import Trade
 
@@ -11,6 +12,7 @@ def disable_database_use(timeframe: str) -> None:
     PairLocks.use_db = False
     PairLocks.timeframe = timeframe
     Trade.use_db = False
+    CustomDataWrapper.use_db = False
 
 
 def enable_database_use() -> None:
@@ -20,6 +22,7 @@ def enable_database_use() -> None:
     PairLocks.use_db = True
     PairLocks.timeframe = ''
     Trade.use_db = True
+    CustomDataWrapper.use_db = True
 
 
 class FtNoDBContext:

freqtrade/rpc/rpc.py

@@ -999,6 +999,32 @@ class RPC:
                 'cancel_order_count': c_count,
             }
 
+    def _rpc_list_custom_data(self, trade_id: int, key: Optional[str]) -> List[Dict[str, Any]]:
+        # Query for trade
+        trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first()
+        if trade is None:
+            return []
+        # Query custom_data
+        custom_data = []
+        if key:
+            data = trade.get_custom_data(key=key)
+            if data:
+                custom_data = [data]
+        else:
+            custom_data = trade.get_all_custom_data()
+        return [
+            {
+                'id': data_entry.id,
+                'ft_trade_id': data_entry.ft_trade_id,
+                'cd_key': data_entry.cd_key,
+                'cd_type': data_entry.cd_type,
+                'cd_value': data_entry.cd_value,
+                'created_at': data_entry.created_at,
+                'updated_at': data_entry.updated_at
+            }
+            for data_entry in custom_data
+        ]
+
     def _rpc_performance(self) -> List[Dict[str, Any]]:
         """
         Handler for performance.

freqtrade/rpc/telegram.py

@@ -33,7 +33,7 @@ from freqtrade.misc import chunks, plural
 from freqtrade.persistence import Trade
 from freqtrade.rpc import RPC, RPCException, RPCHandler
 from freqtrade.rpc.rpc_types import RPCEntryMsg, RPCExitMsg, RPCOrderMsg, RPCSendMsg
-from freqtrade.util import dt_humanize, fmt_coin, round_value
+from freqtrade.util import dt_humanize, fmt_coin, format_date, round_value
 
 
 MAX_MESSAGE_LENGTH = MessageLimit.MAX_TEXT_LENGTH
@@ -243,6 +243,7 @@ class Telegram(RPCHandler):
             CommandHandler('version', self._version),
             CommandHandler('marketdir', self._changemarketdir),
             CommandHandler('order', self._order),
+            CommandHandler('list_custom_data', self._list_custom_data),
         ]
         callbacks = [
             CallbackQueryHandler(self._status_table, pattern='update_status_table'),
@@ -1667,6 +1668,8 @@ class Telegram(RPCHandler):
             "*/marketdir [long | short | even | none]:* `Updates the user managed variable "
             "that represents the current market direction. If no direction is provided `"
             "`the currently set market direction will be output.` \n"
+            "*/list_custom_data <trade_id> <key>:* `List custom_data for Trade ID & Key combo.`\n"
+            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`"
 
             "_Statistics_\n"
             "------------\n"
@@ -1689,7 +1692,7 @@ class Telegram(RPCHandler):
             "*/stats:* `Shows Wins / losses by Sell reason as well as "
             "Avg. holding durations for buys and sells.`\n"
             "*/help:* `This help message`\n"
-            "*/version:* `Show version`"
+            "*/version:* `Show version`\n"
             )
 
         await self._send_msg(message, parse_mode=ParseMode.MARKDOWN)
@@ -1766,6 +1769,53 @@ class Telegram(RPCHandler):
             f"*Current state:* `{val['state']}`"
         )
 
+    @authorized_only
+    async def _list_custom_data(self, update: Update, context: CallbackContext) -> None:
+        """
+        Handler for /list_custom_data <id> <key>.
+        List custom_data for specified trade (and key if supplied).
+        :param bot: telegram bot
+        :param update: message update
+        :return: None
+        """
+        try:
+            if not context.args or len(context.args) == 0:
+                raise RPCException("Trade-id not set.")
+            trade_id = int(context.args[0])
+            key = None if len(context.args) < 2 else str(context.args[1])
+
+            results = self._rpc._rpc_list_custom_data(trade_id, key)
+            messages = []
+            if len(results) > 0:
+                messages.append(
+                    'Found custom-data entr' + ('ies: ' if len(results) > 1 else 'y: ')
+                )
+                for result in results:
+                    lines = [
+                        f"*Key:* `{result['cd_key']}`",
+                        f"*ID:* `{result['id']}`",
+                        f"*Trade ID:* `{result['ft_trade_id']}`",
+                        f"*Type:* `{result['cd_type']}`",
+                        f"*Value:* `{result['cd_value']}`",
+                        f"*Create Date:* `{format_date(result['created_at'])}`",
+                        f"*Update Date:* `{format_date(result['updated_at'])}`"
+                    ]
+                    # Filter empty lines using list-comprehension
+                    messages.append("\n".join([line for line in lines if line]))
+                for msg in messages:
+                    if len(msg) > MAX_MESSAGE_LENGTH:
+                        msg = "Message dropped because length exceeds "
+                        msg += f"maximum allowed characters: {MAX_MESSAGE_LENGTH}"
+                        logger.warning(msg)
+                    await self._send_msg(msg)
+            else:
+                message = f"Didn't find any custom-data entries for Trade ID: `{trade_id}`"
+                message += f" and Key: `{key}`." if key is not None else ""
+                await self._send_msg(message)
+
+        except RPCException as e:
+            await self._send_msg(str(e))
+
     async def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "",
                           reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None:
         if reload_able:

tests/persistence/test_persistence.py

@@ -2099,6 +2099,7 @@ def test_Trade_object_idem():
         'get_mix_tag_performance',
         'get_trading_volume',
         'validate_string_len',
+        'custom_data'
     )
     EXCLUDES2 = ('trades', 'trades_open', 'bt_trades_open_pp', 'bt_open_open_trade_count',
                  'total_profit', 'from_json',)

tests/persistence/test_trade_custom_data.py

@@ -0,0 +1,160 @@
+from copy import deepcopy
+from unittest.mock import MagicMock
+
+import pytest
+
+from freqtrade.data.history.history_utils import get_timerange
+from freqtrade.optimize.backtesting import Backtesting
+from freqtrade.persistence import Trade, disable_database_use, enable_database_use
+from freqtrade.persistence.custom_data import CustomDataWrapper
+from tests.conftest import (EXMS, create_mock_trades_usdt, generate_test_data,
+                            get_patched_freqtradebot, patch_exchange)
+
+
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize("use_db", [True, False])
+def test_trade_custom_data(fee, use_db):
+    if not use_db:
+        disable_database_use('5m')
+    Trade.reset_trades()
+    CustomDataWrapper.reset_custom_data()
+
+    create_mock_trades_usdt(fee, use_db=use_db)
+
+    trade1 = Trade.get_trades_proxy()[0]
+    if not use_db:
+        trade1.id = 1
+
+    assert trade1.get_all_custom_data() == []
+    trade1.set_custom_data('test_str', 'test_value')
+    trade1.set_custom_data('test_int', 1)
+    trade1.set_custom_data('test_float', 1.55)
+    trade1.set_custom_data('test_bool', True)
+    trade1.set_custom_data('test_dict', {'test': 'dict'})
+
+    assert len(trade1.get_all_custom_data()) == 5
+    assert trade1.get_custom_data('test_str') == 'test_value'
+    trade1.set_custom_data('test_str', 'test_value_updated')
+    assert trade1.get_custom_data('test_str') == 'test_value_updated'
+
+    assert trade1.get_custom_data('test_int') == 1
+    assert isinstance(trade1.get_custom_data('test_int'), int)
+
+    assert trade1.get_custom_data('test_float') == 1.55
+    assert isinstance(trade1.get_custom_data('test_float'), float)
+
+    assert trade1.get_custom_data('test_bool') is True
+    assert isinstance(trade1.get_custom_data('test_bool'), bool)
+
+    assert trade1.get_custom_data('test_dict') == {'test': 'dict'}
+    assert isinstance(trade1.get_custom_data('test_dict'), dict)
+    if not use_db:
+        enable_database_use()
+
+
+def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee):
+
+    mocker.patch(f'{EXMS}.get_rate', return_value=0.50)
+    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=None)
+    default_conf_usdt["minimal_roi"] = {"0":  100}
+
+    freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
+    create_mock_trades_usdt(fee)
+
+    trade1 = Trade.get_trades_proxy(pair='ADA/USDT')[0]
+    trade1.set_custom_data('test_str', 'test_value')
+    trade1.set_custom_data('test_int', 1)
+
+    def custom_exit(pair, trade, **kwargs):
+
+        if pair == 'ADA/USDT':
+            custom_val = trade.get_custom_data('test_str')
+            custom_val_i = trade.get_custom_data('test_int')
+
+            return f"{custom_val}_{custom_val_i}"
+
+    freqtrade.strategy.custom_exit = custom_exit
+    ff_spy = mocker.spy(freqtrade.strategy, 'custom_exit')
+    trades = Trade.get_open_trades()
+    freqtrade.exit_positions(trades)
+    Trade.commit()
+
+    trade_after = Trade.get_trades_proxy(pair='ADA/USDT')[0]
+    assert trade_after.get_custom_data('test_str') == 'test_value'
+    assert trade_after.get_custom_data('test_int') == 1
+    # 2 open pairs eligible for exit
+    assert ff_spy.call_count == 2
+
+    assert trade_after.exit_reason == 'test_value_1'
+
+
+def test_trade_custom_data_strategy_backtest_compat(mocker, default_conf_usdt, fee):
+
+    mocker.patch(f'{EXMS}.get_fee', fee)
+    mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=10)
+    mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float('inf'))
+    mocker.patch(f"{EXMS}.get_max_leverage", return_value=10)
+    mocker.patch(f"{EXMS}.get_maintenance_ratio_and_amt", return_value=(0.1, 0.1))
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting._run_funding_fees')
+
+    patch_exchange(mocker)
+    default_conf_usdt.update({
+        "stake_amount": 100.0,
+        "max_open_trades": 2,
+        "dry_run_wallet": 1000.0,
+        "strategy": "StrategyTestV3",
+        "trading_mode": "futures",
+        "margin_mode": "isolated",
+        "stoploss": -2,
+        "minimal_roi": {"0":  100},
+    })
+    default_conf_usdt['pairlists'] = [{'method': 'StaticPairList', 'allow_inactive': True}]
+    backtesting = Backtesting(default_conf_usdt)
+
+    df = generate_test_data(default_conf_usdt['timeframe'], 100, '2022-01-01 00:00:00+00:00')
+
+    pair_exp = 'XRP/USDT:USDT'
+
+    def custom_exit(pair, trade, **kwargs):
+        custom_val = trade.get_custom_data('test_str')
+        custom_val_i = trade.get_custom_data('test_int', 0)
+
+        if pair == pair_exp:
+            trade.set_custom_data('test_str', 'test_value')
+            trade.set_custom_data('test_int', custom_val_i + 1)
+
+        if custom_val_i >= 2:
+            return f"{custom_val}_{custom_val_i}"
+
+    backtesting._set_strategy(backtesting.strategylist[0])
+    processed = backtesting.strategy.advise_all_indicators({
+        pair_exp: df,
+        'BTC/USDT:USDT': df,
+    })
+
+    def fun(dataframe, *args, **kwargs):
+        dataframe.loc[dataframe.index == 50, 'enter_long'] = 1
+        return dataframe
+
+    backtesting.strategy.advise_entry = fun
+    backtesting.strategy.leverage = MagicMock(return_value=1)
+    backtesting.strategy.custom_exit = custom_exit
+    ff_spy = mocker.spy(backtesting.strategy, 'custom_exit')
+
+    min_date, max_date = get_timerange(processed)
+
+    result = backtesting.backtest(
+        processed=deepcopy(processed),
+        start_date=min_date,
+        end_date=max_date,
+    )
+    results = result['results']
+    assert not results.empty
+    assert len(results) == 2
+    assert results['pair'][0] == pair_exp
+    assert results['pair'][1] == 'BTC/USDT:USDT'
+    assert results['exit_reason'][0] == 'test_value_2'
+    assert results['exit_reason'][1] == 'exit_signal'
+
+    assert ff_spy.call_count == 7
+    Backtesting.cleanup()

tests/rpc/test_rpc_telegram.py

@@ -150,7 +150,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None:
                    "['stopbuy', 'stopentry'], ['whitelist'], ['blacklist'], "
                    "['bl_delete', 'blacklist_delete'], "
                    "['logs'], ['edge'], ['health'], ['help'], ['version'], ['marketdir'], "
-                   "['order']]")
+                   "['order'], ['list_custom_data']]")
 
     assert log_has(message_str, caplog)
 
@@ -2657,3 +2657,49 @@ async def test_change_market_direction(default_conf, mocker, update) -> None:
     context.args = ["invalid"]
     await telegram._changemarketdir(update, context)
     assert telegram._rpc._freqtrade.strategy.market_direction == MarketDirection.LONG
+
+
+async def test_telegram_list_custom_data(default_conf_usdt, update, ticker, fee, mocker) -> None:
+
+    mocker.patch.multiple(
+        EXMS,
+        fetch_ticker=ticker,
+        get_fee=fee,
+    )
+    telegram, _freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt)
+
+    # Create some test data
+    create_mock_trades_usdt(fee)
+    # No trade id
+    context = MagicMock()
+    await telegram._list_custom_data(update=update, context=context)
+    assert msg_mock.call_count == 1
+    assert 'Trade-id not set.' in msg_mock.call_args_list[0][0][0]
+    msg_mock.reset_mock()
+
+    #
+    context.args = ['1']
+    await telegram._list_custom_data(update=update, context=context)
+    assert msg_mock.call_count == 1
+    assert (
+        "Didn't find any custom-data entries for Trade ID: `1`" in msg_mock.call_args_list[0][0][0]
+    )
+    msg_mock.reset_mock()
+
+    # Add some custom data
+    trade1 = Trade.get_trades_proxy()[0]
+    trade1.set_custom_data('test_int', 1)
+    trade1.set_custom_data('test_dict', {'test': 'dict'})
+    Trade.commit()
+    context.args = [f"{trade1.id}"]
+    await telegram._list_custom_data(update=update, context=context)
+    assert msg_mock.call_count == 3
+    assert "Found custom-data entries: " in msg_mock.call_args_list[0][0][0]
+    assert (
+        "*Key:* `test_int`\n*ID:* `1`\n*Trade ID:* `1`\n*Type:* `int`\n"
+        "*Value:* `1`\n*Create Date:*") in msg_mock.call_args_list[1][0][0]
+    assert (
+        '*Key:* `test_dict`\n*ID:* `2`\n*Trade ID:* `1`\n*Type:* `dict`\n'
+        '*Value:* `{"test": "dict"}`\n*Create Date:* `') in msg_mock.call_args_list[2][0][0]
+
+    msg_mock.reset_mock()