diff --git a/docs/exchanges.md b/docs/exchanges.md index fb3049ba5..63819393e 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -55,7 +55,7 @@ This configuration enables kraken, as well as rate-limiting to avoid bans from t ## Binance !!! Warning "Server location and geo-ip restrictions" - Please be aware that binance restrict api access regarding the server country. The currents and non exhaustive countries blocked are United States, Malaysia (Singapour), Ontario (Canada). Please go to [binance terms > b. Eligibility](https://www.binance.com/en/terms) to find up to date list. + Please be aware that Binance restricts API access regarding the server country. The current and non-exhaustive countries blocked are Canada, Malaysia, Netherlands and United States. Please go to [binance terms > b. Eligibility](https://www.binance.com/en/terms) to find up to date list. Binance supports [time_in_force](configuration.md#understand-order_time_in_force). diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 9cdcc9bca..74746a96b 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -376,7 +376,7 @@ If the trading range over the last 10 days is <1% or >99%, remove the pair from "lookback_days": 10, "min_rate_of_change": 0.01, "max_rate_of_change": 0.99, - "refresh_period": 1440 + "refresh_period": 86400 } ] ``` @@ -431,7 +431,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01, - "refresh_period": 1440 + "refresh_period": 86400 }, { "method": "VolatilityFilter", diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index e2292e640..4d66ebb5e 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.4.4 mkdocs==1.5.2 -mkdocs-material==9.2.8 +mkdocs-material==9.3.1 mdx_truly_sane_lists==1.3 pymdown-extensions==10.3 jinja2==3.1.2 diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 5ce898dbb..e42aa39d2 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -510,6 +510,9 @@ Each of these methods are called right before placing an order on the exchange. !!! Note If your custom pricing function return None or an invalid value, price will fall back to `proposed_rate`, which is based on the regular pricing configuration. +!!! Note + Using custom_entry_price, the Trade object will be available as soon as the first entry order associated with the trade is created, for the first entry, `trade` parameter value will be `None`. + ### Custom order entry and exit price example ``` python @@ -520,7 +523,7 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Optional['Trade'], current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, @@ -823,7 +826,7 @@ class DigDeeperStrategy(IStrategy): """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. - This means extra buy or sell orders with additional fees. + This means extra entry or exit orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -832,8 +835,9 @@ class DigDeeperStrategy(IStrategy): :param trade: trade object. :param current_time: datetime object, containing the current datetime - :param current_rate: Current buy rate. - :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param current_rate: Current entry rate (same as current_entry_profit) + :param current_profit: Current profit (as ratio), calculated based on current_rate + (same as current_entry_profit). :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). :param current_entry_rate: Current rate using entry pricing. diff --git a/docs/strategy_migration.md b/docs/strategy_migration.md index d4d5f0068..9e6f56e49 100644 --- a/docs/strategy_migration.md +++ b/docs/strategy_migration.md @@ -280,7 +280,7 @@ After: ``` python hl_lines="3" class AwesomeStrategy(IStrategy): - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Optional[Trade], current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: return proposed_rate ``` diff --git a/freqtrade/constants.py b/freqtrade/constants.py index ecd9f6e5d..1bd51c395 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -77,7 +77,8 @@ DL_DATA_TIMEFRAMES = ['1m', '5m'] ENV_VAR_PREFIX = 'FREQTRADE__' -NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired') +CANCELED_EXCHANGE_STATES = ('cancelled', 'canceled', 'expired') +NON_OPEN_EXCHANGE_STATES = CANCELED_EXCHANGE_STATES + ('closed',) # Define decimals per coin for outputs # Only used for outputs. diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ddb00ecef..fac02509b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1420,7 +1420,7 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e - def __fetch_orders_emulate(self, pair: str, since_ms: int) -> List[Dict]: + def _fetch_orders_emulate(self, pair: str, since_ms: int) -> List[Dict]: orders = [] if self.exchange_has('fetchClosedOrders'): orders = self._api.fetch_closed_orders(pair, since=since_ms) @@ -1450,9 +1450,9 @@ class Exchange: except ccxt.NotSupported: # Some exchanges don't support fetchOrders # attempt to fetch open and closed orders separately - orders = self.__fetch_orders_emulate(pair, since_ms) + orders = self._fetch_orders_emulate(pair, since_ms) else: - orders = self.__fetch_orders_emulate(pair, since_ms) + orders = self._fetch_orders_emulate(pair, since_ms) self._log_exchange_response('fetch_orders', orders) orders = [self._order_contracts_to_amount(o) for o in orders] return orders diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index c0629240d..b99911994 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -1,4 +1,5 @@ import logging +from datetime import timedelta from typing import Any, Dict, List, Optional, Tuple import ccxt @@ -10,6 +11,7 @@ from freqtrade.exceptions import (DDosProtection, OperationalException, Retryabl from freqtrade.exchange import Exchange, date_minus_candles from freqtrade.exchange.common import retrier from freqtrade.misc import safe_value_fallback2 +from freqtrade.util import dt_now, dt_ts logger = logging.getLogger(__name__) @@ -186,7 +188,7 @@ class Okx(Exchange): def _convert_stop_order(self, pair: str, order_id: str, order: Dict) -> Dict: if ( - order['status'] == 'closed' + order.get('status', 'open') == 'closed' and (real_order_id := order.get('info', {}).get('ordId')) is not None ): # Once a order triggered, we fetch the regular followup order. @@ -240,3 +242,18 @@ class Okx(Exchange): pair=pair, params=params1, ) + + def _fetch_orders_emulate(self, pair: str, since_ms: int) -> List[Dict]: + orders = [] + + orders = self._api.fetch_closed_orders(pair, since=since_ms) + if (since_ms < dt_ts(dt_now() - timedelta(days=6, hours=23))): + # Regular fetch_closed_orders only returns 7 days of data. + # Force usage of "archive" endpoint, which returns 3 months of data. + params = {'method': 'privateGetTradeOrdersHistoryArchive'} + orders_hist = self._api.fetch_closed_orders(pair, since=since_ms, params=params) + orders.extend(orders_hist) + + orders_open = self._api.fetch_open_orders(pair, since=since_ms) + orders.extend(orders_open) + return orders diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index b6ded83b1..0306282c0 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -263,23 +263,46 @@ class FreqaiDataDrawer: self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy() return - def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None: + def set_initial_return_values(self, pair: str, + pred_df: DataFrame, + dataframe: DataFrame + ) -> None: """ Set the initial return values to the historical predictions dataframe. This avoids needing to repredict on historical candles, and also stores historical predictions despite retrainings (so stored predictions are true predictions, not just inferencing on trained - data) + data). + + We also aim to keep the date from historical predictions so that the FreqUI displays + zeros during any downtime (between FreqAI reloads). """ - hist_df = self.historic_predictions - len_diff = len(hist_df[pair].index) - len(pred_df.index) - if len_diff < 0: - df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]], - ignore_index=True, keys=hist_df[pair].keys()) + new_pred = pred_df.copy() + # set new_pred values to nans (we want to signal to user that there was nothing + # historically made during downtime. The newest pred will get appeneded later in + # append_model_predictions) + new_pred.iloc[:, :] = np.nan + new_pred["date_pred"] = dataframe["date"] + hist_preds = self.historic_predictions[pair].copy() + + # find the closest common date between new_pred and historic predictions + # and cut off the new_pred dataframe at that date + common_dates = pd.merge(new_pred, hist_preds, on="date_pred", how="inner") + if len(common_dates.index) > 0: + new_pred = new_pred.iloc[len(common_dates):] else: - df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True) + logger.warning("No common dates found between new predictions and historic " + "predictions. You likely left your FreqAI instance offline " + f"for more than {len(dataframe.index)} candles.") + + df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys()) + # remove last row because we will append that later in append_model_predictions() + df_concat = df_concat.iloc[:-1] + # any missing values will get zeroed out so users can see the exact + # downtime in FreqUI df_concat = df_concat.fillna(0) - self.model_return_values[pair] = df_concat + self.historic_predictions[pair] = df_concat + self.model_return_values[pair] = df_concat.tail(len(dataframe.index)).reset_index(drop=True) def append_model_predictions(self, pair: str, predictions: DataFrame, do_preds: NDArray[np.int_], diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 7d4bf39ca..3d1ed7849 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -244,6 +244,14 @@ class FreqaiDataKitchen: f"{self.pair}: dropped {len(unfiltered_df) - len(filtered_df)} training points" f" due to NaNs in populated dataset {len(unfiltered_df)}." ) + if len(unfiltered_df) == 0 and not self.live: + raise OperationalException( + f"{self.pair}: all training data dropped due to NaNs. " + "You likely did not download enough training data prior " + "to your backtest timerange. Hint:\n" + "https://www.freqtrade.io/en/stable/freqai-running/" + "#downloading-data-to-cover-the-full-backtest-period" + ) if (1 - len(filtered_df) / len(unfiltered_df)) > 0.1 and self.live: worst_indicator = str(unfiltered_df.count().idxmin()) logger.warning( diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index efae6d060..33d23aa73 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -453,7 +453,7 @@ class IFreqaiModel(ABC): pred_df, do_preds = self.predict(dataframe, dk) if pair not in self.dd.historic_predictions: self.set_initial_historic_predictions(pred_df, dk, pair, dataframe) - self.dd.set_initial_return_values(pair, pred_df) + self.dd.set_initial_return_values(pair, pred_df, dataframe) dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe) return @@ -645,11 +645,11 @@ class IFreqaiModel(ABC): If the user reuses an identifier on a subsequent instance, this function will not be called. In that case, "real" predictions will be appended to the loaded set of historic predictions. - :param df: DataFrame = the dataframe containing the training feature data - :param model: Any = A model which was `fit` using a common library such as - catboost or lightgbm + :param pred_df: DataFrame = the dataframe containing the predictions coming + out of a model :param dk: FreqaiDataKitchen = object containing methods for data analysis :param pair: str = current pair + :param strat_df: DataFrame = dataframe coming from strategy """ self.dd.historic_predictions[pair] = pred_df diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1a24009b3..3033a9daa 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -456,31 +456,42 @@ class FreqtradeBot(LoggingMixin): Only used balance disappeared, which would make exiting impossible. """ try: - orders = self.exchange.fetch_orders(trade.pair, trade.open_date_utc) + orders = self.exchange.fetch_orders( + trade.pair, trade.open_date_utc - timedelta(seconds=10)) + prev_exit_reason = trade.exit_reason + prev_trade_state = trade.is_open for order in orders: trade_order = [o for o in trade.orders if o.order_id == order['id']] - if trade_order: - continue - logger.info(f"Found previously unknown order {order['id']} for {trade.pair}.") - order_obj = Order.parse_from_ccxt_object(order, trade.pair, order['side']) - order_obj.order_filled_date = datetime.fromtimestamp( - safe_value_fallback(order, 'lastTradeTimestamp', 'timestamp') // 1000, - tz=timezone.utc) - trade.orders.append(order_obj) - prev_exit_reason = trade.exit_reason - trade.exit_reason = ExitType.SOLD_ON_EXCHANGE.value - self.update_trade_state(trade, order['id'], order) + if trade_order: + # We knew this order, but didn't have it updated properly + order_obj = trade_order[0] + else: + logger.info(f"Found previously unknown order {order['id']} for {trade.pair}.") + + order_obj = Order.parse_from_ccxt_object(order, trade.pair, order['side']) + order_obj.order_filled_date = datetime.fromtimestamp( + safe_value_fallback(order, 'lastTradeTimestamp', 'timestamp') // 1000, + tz=timezone.utc) + trade.orders.append(order_obj) + Trade.commit() + trade.exit_reason = ExitType.SOLD_ON_EXCHANGE.value + + self.update_trade_state(trade, order['id'], order, send_msg=False) logger.info(f"handled order {order['id']}") - if not trade.is_open: - # Trade was just closed - trade.close_date = order_obj.order_filled_date - Trade.commit() - break - else: - trade.exit_reason = prev_exit_reason - Trade.commit() + + # Refresh trade from database + Trade.session.refresh(trade) + if not trade.is_open: + # Trade was just closed + trade.close_date = trade.date_last_filled_utc + self.order_close_notify(trade, order_obj, + order_obj.ft_order_side == 'stoploss', + send_msg=prev_trade_state != trade.is_open) + else: + trade.exit_reason = prev_exit_reason + Trade.commit() except ExchangeError: logger.warning("Error finding onexchange order.") @@ -927,7 +938,8 @@ class FreqtradeBot(LoggingMixin): # Don't call custom_entry_price in order-adjust scenario custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, default_retval=enter_limit_requested)( - pair=pair, current_time=datetime.now(timezone.utc), + pair=pair, trade=trade, + current_time=datetime.now(timezone.utc), proposed_rate=enter_limit_requested, entry_tag=entry_tag, side=trade_side, ) @@ -1352,18 +1364,21 @@ class FreqtradeBot(LoggingMixin): self.handle_cancel_enter(trade, order, order_id, reason) else: canceled = self.handle_cancel_exit(trade, order, order_id, reason) - canceled_count = trade.get_exit_order_count() + canceled_count = trade.get_canceled_exit_order_count() max_timeouts = self.config.get('unfilledtimeout', {}).get('exit_timeout_count', 0) - if canceled and max_timeouts > 0 and canceled_count >= max_timeouts: - logger.warning(f'Emergency exiting trade {trade}, as the exit order ' - f'timed out {max_timeouts} times.') - self.emergency_exit(trade, order['price']) + if (canceled and max_timeouts > 0 and canceled_count >= max_timeouts): + logger.warning(f"Emergency exiting trade {trade}, as the exit order " + f"timed out {max_timeouts} times. force selling {order['amount']}.") + self.emergency_exit(trade, order['price'], order['amount']) - def emergency_exit(self, trade: Trade, price: float) -> None: + def emergency_exit( + self, trade: Trade, price: float, sub_trade_amt: Optional[float] = None) -> None: try: self.execute_trade_exit( trade, price, - exit_check=ExitCheckTuple(exit_type=ExitType.EMERGENCY_EXIT)) + exit_check=ExitCheckTuple(exit_type=ExitType.EMERGENCY_EXIT), + sub_trade_amt=sub_trade_amt + ) except DependencyException as exception: logger.warning( f'Unable to emergency exit trade {trade.pair}: {exception}') @@ -1838,7 +1853,7 @@ class FreqtradeBot(LoggingMixin): def update_trade_state( self, trade: Trade, order_id: Optional[str], - action_order: Optional[Dict[str, Any]] = None, + action_order: Optional[Dict[str, Any]] = None, *, stoploss_order: bool = False, send_msg: bool = True) -> bool: """ Checks trades with open orders and updates the amount if necessary @@ -1875,7 +1890,7 @@ class FreqtradeBot(LoggingMixin): self.handle_order_fee(trade, order_obj, order) - trade.update_trade(order_obj) + trade.update_trade(order_obj, not send_msg) trade = self._update_trade_after_fill(trade, order_obj) Trade.commit() diff --git a/freqtrade/main.py b/freqtrade/main.py index a10620498..05e5409ad 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -11,8 +11,8 @@ from freqtrade.util.gc_setup import gc_set_threshold # check min. python version -if sys.version_info < (3, 8): # pragma: no cover - sys.exit("Freqtrade requires Python version >= 3.8") +if sys.version_info < (3, 9): # pragma: no cover + sys.exit("Freqtrade requires Python version >= 3.9") from freqtrade import __version__ from freqtrade.commands import Arguments diff --git a/freqtrade/misc.py b/freqtrade/misc.py index f8d730fae..cbebf99eb 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -156,7 +156,7 @@ def round_dict(d, n): return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()} -def safe_value_fallback(obj: dict, key1: str, key2: str, default_value=None): +def safe_value_fallback(obj: dict, key1: str, key2: Optional[str] = None, default_value=None): """ Search a value in obj, return this if it's not None. Then search key2 in obj - return that if it's not none - then use default_value. @@ -165,7 +165,7 @@ def safe_value_fallback(obj: dict, key1: str, key2: str, default_value=None): if key1 in obj and obj[key1] is not None: return obj[key1] else: - if key2 in obj and obj[key2] is not None: + if key2 and key2 in obj and obj[key2] is not None: return obj[key2] return default_value diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4b267b315..6db996589 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -738,7 +738,9 @@ class Backtesting: if order_type == 'limit': new_rate = strategy_safe_wrapper(self.strategy.custom_entry_price, default_retval=propose_rate)( - pair=pair, current_time=current_time, + pair=pair, + trade=trade, # type: ignore[arg-type] + current_time=current_time, proposed_rate=propose_rate, entry_tag=entry_tag, side=direction, ) # default value is the open rate diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 22444bb49..8efce76d1 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -13,15 +13,17 @@ from sqlalchemy import (Enum, Float, ForeignKey, Integer, ScalarResult, Select, from sqlalchemy.orm import Mapped, lazyload, mapped_column, relationship, validates from typing_extensions import Self -from freqtrade.constants import (CUSTOM_TAG_MAX_LENGTH, DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, - NON_OPEN_EXCHANGE_STATES, BuySell, LongShort) +from freqtrade.constants import (CANCELED_EXCHANGE_STATES, CUSTOM_TAG_MAX_LENGTH, + DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES, + BuySell, LongShort) from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precision, price_to_precision) from freqtrade.leverage import interest +from freqtrade.misc import safe_value_fallback from freqtrade.persistence.base import ModelBase, SessionType -from freqtrade.util import FtPrecise, dt_now +from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts logger = logging.getLogger(__name__) @@ -176,7 +178,9 @@ class Order(ModelBase): # (represents the funding fee since the last order) self.funding_fee = self.trade.funding_fees if (order.get('filled', 0.0) or 0.0) > 0 and not self.order_filled_date: - self.order_filled_date = datetime.now(timezone.utc) + self.order_filled_date = dt_from_ts( + safe_value_fallback(order, 'lastTradeTimestamp', default_value=dt_ts()) + ) self.order_update_date = datetime.now(timezone.utc) def to_ccxt_object(self, stopPriceName: str = 'stopPrice') -> Dict[str, Any]: @@ -430,13 +434,20 @@ class LocalTrade: return self.amount @property - def date_last_filled_utc(self) -> datetime: + def _date_last_filled_utc(self) -> Optional[datetime]: """ Date of the last filled order""" orders = self.select_filled_orders() - if not orders: + if orders: + return max(o.order_filled_utc for o in orders if o.order_filled_utc) + return None + + @property + def date_last_filled_utc(self) -> datetime: + """ Date of the last filled order - or open_date if no orders are filled""" + dt_last_filled = self._date_last_filled_utc + if not dt_last_filled: return self.open_date_utc - return max([self.open_date_utc, - max(o.order_filled_utc for o in orders if o.order_filled_utc)]) + return max([self.open_date_utc, dt_last_filled]) @property def open_date_utc(self): @@ -617,8 +628,9 @@ class LocalTrade: 'amount_precision': self.amount_precision, 'price_precision': self.price_precision, 'precision_mode': self.precision_mode, - 'orders': orders_json, + 'contract_size': self.contract_size, 'has_open_orders': self.has_open_orders, + 'orders': orders_json, } @staticmethod @@ -716,7 +728,7 @@ class LocalTrade: f"Trailing stoploss saved us: " f"{float(self.stop_loss) - float(self.initial_stop_loss or 0.0):.8f}.") - def update_trade(self, order: Order) -> None: + def update_trade(self, order: Order, recalculating: bool = False) -> None: """ Updates this entity with amount and actual open/close rates. :param order: order retrieved by exchange.fetch_order() @@ -758,8 +770,9 @@ class LocalTrade: self.precision_mode, self.contract_size) if ( isclose(order.safe_amount_after_fee, amount_tr, abs_tol=MATH_CLOSE_PREC) - or order.safe_amount_after_fee > amount_tr + or (not recalculating and order.safe_amount_after_fee > amount_tr) ): + # When recalculating a trade, only comming out to 0 can force a close self.close(order.safe_price) else: self.recalc_trade_from_orders() @@ -772,7 +785,7 @@ class LocalTrade: and marks trade as closed """ self.close_rate = rate - self.close_date = self.close_date or datetime.utcnow() + self.close_date = self.close_date or self._date_last_filled_utc or dt_now() self.is_open = False self.exit_order_status = 'closed' self.recalc_trade_from_orders(is_closing=True) @@ -812,12 +825,13 @@ class LocalTrade: def update_order(self, order: Dict) -> None: Order.update_orders(self.orders, order) - def get_exit_order_count(self) -> int: + def get_canceled_exit_order_count(self) -> int: """ Get amount of failed exiting orders assumes full exits. """ - return len([o for o in self.orders if o.ft_order_side == self.exit_side]) + return len([o for o in self.orders if o.ft_order_side == self.exit_side + and o.status in CANCELED_EXCHANGE_STATES]) def _calc_open_trade_value(self, amount: float, open_rate: float) -> float: """ @@ -1776,6 +1790,10 @@ class Trade(ModelBase, LocalTrade): is_short=data["is_short"], trading_mode=data["trading_mode"], funding_fees=data["funding_fees"], + amount_precision=data.get('amount_precision', None), + price_precision=data.get('price_precision', None), + precision_mode=data.get('precision_mode', None), + contract_size=data.get('contract_size', None), ) for order in data["orders"]: diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index f294b882b..f4625f572 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -30,7 +30,7 @@ class RangeStabilityFilter(IPairList): self._days = pairlistconfig.get('lookback_days', 10) self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01) self._max_rate_of_change = pairlistconfig.get('max_rate_of_change') - self._refresh_period = pairlistconfig.get('refresh_period', 1440) + self._refresh_period = pairlistconfig.get('refresh_period', 86400) self._def_candletype = self._config['candle_type_def'] self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 43d2a0baf..5cdbb6bf6 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -395,7 +395,8 @@ class IStrategy(ABC, HyperStrategyMixin): """ return self.stoploss - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Optional[Trade], + current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: """ Custom entry price logic, returning the new entry price. @@ -405,6 +406,7 @@ class IStrategy(ABC, HyperStrategyMixin): When not implemented by a strategy, returns None, orderbook is used to set entry price :param pair: Pair that's currently analyzed + :param trade: trade object (None for initial entries). :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. @@ -513,7 +515,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. - This means extra buy or sell orders with additional fees. + This means extra entry or exit orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -522,8 +524,9 @@ class IStrategy(ABC, HyperStrategyMixin): :param trade: trade object. :param current_time: datetime object, containing the current datetime - :param current_rate: Current buy rate. - :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param current_rate: Current entry rate (same as current_entry_profit) + :param current_profit: Current profit (as ratio), calculated based on current_rate + (same as current_entry_profit). :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). :param current_entry_rate: Current rate using entry pricing. diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 084cf2e89..e64570b9e 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -31,7 +31,7 @@ class FreqaiExampleStrategy(IStrategy): plot_config = { "main_plot": {}, "subplots": { - "&-s_close": {"prediction": {"color": "blue"}}, + "&-s_close": {"&-s_close": {"color": "blue"}}, "do_predict": { "do_predict": {"color": "brown"}, }, diff --git a/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 index 95c6df2ea..6fad129c7 100644 --- a/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 @@ -13,7 +13,8 @@ def bot_loop_start(self, current_time: datetime, **kwargs) -> None: """ pass -def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: float, +def custom_entry_price(self, pair: str, trade: Optional['Trade'], + current_time: 'datetime', proposed_rate: float, entry_tag: 'Optional[str]', side: str, **kwargs) -> float: """ Custom entry price logic, returning the new entry price. @@ -23,6 +24,7 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: When not implemented by a strategy, returns None, orderbook is used to set entry price :param pair: Pair that's currently analyzed + :param trade: trade object (None for initial entries). :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. @@ -257,7 +259,7 @@ def adjust_trade_position(self, trade: 'Trade', current_time: datetime, """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. - This means extra buy or sell orders with additional fees. + This means extra entry or exit orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -266,8 +268,9 @@ def adjust_trade_position(self, trade: 'Trade', current_time: datetime, :param trade: trade object. :param current_time: datetime object, containing the current datetime - :param current_rate: Current buy rate. - :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param current_rate: Current entry rate (same as current_entry_profit) + :param current_profit: Current profit (as ratio), calculated based on current_rate + (same as current_entry_profit). :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). :param current_entry_rate: Current rate using entry pricing. @@ -276,8 +279,8 @@ def adjust_trade_position(self, trade: 'Trade', current_time: datetime, :param current_exit_profit: Current profit using exit pricing. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: Stake amount to adjust your trade, - Positive values to increase position, Negative values to decrease position. - Return None for no action. + Positive values to increase position, Negative values to decrease position. + Return None for no action. """ return None diff --git a/pyproject.toml b/pyproject.toml index 40c0e2005..cd0c65916 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ ignore = ["freqtrade/vendor/**"] line-length = 100 extend-exclude = [".env", ".venv"] target-version = "py38" +# Exclude UP036 as it's causing the "exit if < 3.9" to fail. extend-select = [ "C90", # mccabe # "N", # pep8-naming diff --git a/requirements-dev.txt b/requirements-dev.txt index e779429a0..f19c932fd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.0.287 +ruff==0.0.290 mypy==1.5.1 pre-commit==3.4.0 pytest==7.4.2 diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 8f690f957..0c11f8a09 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -6,7 +6,7 @@ scikit-learn==1.1.3 joblib==1.3.2 catboost==1.2.1; 'arm' not in platform_machine -lightgbm==4.0.0 -xgboost==1.7.6 +lightgbm==4.1.0 +xgboost==2.0.0 tensorboard==2.14.0 datasieve==0.1.7 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 4cc7cc4c7..06f8ddbaf 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,4 +5,4 @@ scipy==1.11.2 scikit-learn==1.1.3 scikit-optimize==0.9.0 -filelock==3.12.3 +filelock==3.12.4 diff --git a/requirements-plot.txt b/requirements-plot.txt index 9a8c596ad..b2ec35539 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.16.1 +plotly==5.17.0 diff --git a/requirements.txt b/requirements.txt index f91a8d2b5..fc85cd05c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.25.2 +numpy==1.26.0 pandas==2.0.3 pandas-ta==0.3.14b @@ -23,14 +23,14 @@ jinja2==3.1.2 tables==3.8.0 blosc==1.11.1 joblib==1.3.2 -rich==13.5.2 +rich==13.5.3 pyarrow==13.0.0; platform_machine != 'armv7l' # find first, C search in arrays py_find_1st==1.1.5 # Load ticker files 30% faster -python-rapidjson==1.10 +python-rapidjson==1.11 # Properly format api responses orjson==3.9.7 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 73baa15f2..8826fb876 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -24,7 +24,7 @@ from tests.conftest import (EXMS, generate_test_data_raw, get_mock_coro, get_pat # Make sure to always keep one exchange here which is NOT subclassed!! -EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate', 'kucoin', 'bybit'] +EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate', 'kucoin', 'bybit', 'okx'] get_entry_rate_data = [ ('other', 20, 19, 10, 0.0, 20), # Full ask side @@ -1312,8 +1312,11 @@ def test_create_order(default_conf, mocker, side, ordertype, rate, marketprice, leverage=3.0 ) - assert exchange._set_leverage.call_count == 1 - assert exchange.set_margin_mode.call_count == 1 + if exchange_name != 'okx': + assert exchange._set_leverage.call_count == 1 + assert exchange.set_margin_mode.call_count == 1 + else: + assert api_mock.set_leverage.call_count == 1 assert order['amount'] == 0.01 @@ -1677,7 +1680,10 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): api_mock.fetch_closed_orders = MagicMock(return_value=[limit_order['buy']]) mocker.patch(f'{EXMS}.exchange_has', return_value=True) - start_time = datetime.now(timezone.utc) - timedelta(days=5) + start_time = datetime.now(timezone.utc) - timedelta(days=20) + expected = 1 + if exchange_name == 'bybit': + expected = 3 exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) # Not available in dry-run @@ -1687,10 +1693,10 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) res = exchange.fetch_orders('mocked', start_time) - assert api_mock.fetch_orders.call_count == 1 + assert api_mock.fetch_orders.call_count == expected assert api_mock.fetch_open_orders.call_count == 0 assert api_mock.fetch_closed_orders.call_count == 0 - assert len(res) == 2 + assert len(res) == 2 * expected res = exchange.fetch_orders('mocked', start_time) @@ -1704,13 +1710,17 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): if endpoint == 'fetchOpenOrders': return True + if exchange_name == 'okx': + # Special OKX case is tested separately + return + mocker.patch(f'{EXMS}.exchange_has', has_resp) # happy path without fetchOrders - res = exchange.fetch_orders('mocked', start_time) + exchange.fetch_orders('mocked', start_time) assert api_mock.fetch_orders.call_count == 0 - assert api_mock.fetch_open_orders.call_count == 1 - assert api_mock.fetch_closed_orders.call_count == 1 + assert api_mock.fetch_open_orders.call_count == expected + assert api_mock.fetch_closed_orders.call_count == expected mocker.patch(f'{EXMS}.exchange_has', return_value=True) @@ -1723,11 +1733,11 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): api_mock.fetch_open_orders.reset_mock() api_mock.fetch_closed_orders.reset_mock() - res = exchange.fetch_orders('mocked', start_time) + exchange.fetch_orders('mocked', start_time) - assert api_mock.fetch_orders.call_count == 1 - assert api_mock.fetch_open_orders.call_count == 1 - assert api_mock.fetch_closed_orders.call_count == 1 + assert api_mock.fetch_orders.call_count == expected + assert api_mock.fetch_open_orders.call_count == expected + assert api_mock.fetch_closed_orders.call_count == expected def test_fetch_trading_fees(default_conf, mocker): @@ -2044,7 +2054,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ ) # Required candles candles = (end_ts - start_ts) / 300_000 - exp = candles // exchange.ohlcv_candle_limit('5m', CandleType.SPOT) + 1 + exp = candles // exchange.ohlcv_candle_limit('5m', candle_type, start_ts) + 1 # Depending on the exchange, this should be called between 1 and 6 times. assert exchange._api_async.fetch_ohlcv.call_count == exp @@ -3122,25 +3132,28 @@ def test_cancel_stoploss_order(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name): default_conf['dry_run'] = False + mock_prefix = 'freqtrade.exchange.gate.Gate' + if exchange_name == 'okx': + mock_prefix = 'freqtrade.exchange.okx.Okx' mocker.patch(f'{EXMS}.fetch_stoploss_order', return_value={'for': 123}) - mocker.patch('freqtrade.exchange.gate.Gate.fetch_stoploss_order', return_value={'for': 123}) + mocker.patch(f'{mock_prefix}.fetch_stoploss_order', return_value={'for': 123}) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) res = {'fee': {}, 'status': 'canceled', 'amount': 1234} mocker.patch(f'{EXMS}.cancel_stoploss_order', return_value=res) - mocker.patch('freqtrade.exchange.gate.Gate.cancel_stoploss_order', return_value=res) + mocker.patch(f'{mock_prefix}.cancel_stoploss_order', return_value=res) co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) assert co == res mocker.patch(f'{EXMS}.cancel_stoploss_order', return_value='canceled') - mocker.patch('freqtrade.exchange.gate.Gate.cancel_stoploss_order', return_value='canceled') + mocker.patch(f'{mock_prefix}.cancel_stoploss_order', return_value='canceled') # Fall back to fetch_stoploss_order co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) assert co == {'for': 123} exc = InvalidOrderException("") mocker.patch(f'{EXMS}.fetch_stoploss_order', side_effect=exc) - mocker.patch('freqtrade.exchange.gate.Gate.fetch_stoploss_order', side_effect=exc) + mocker.patch(f'{mock_prefix}.fetch_stoploss_order', side_effect=exc) co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) assert co['amount'] == 555 assert co == {'fee': {}, 'status': 'canceled', 'amount': 555, 'info': {}} @@ -3148,7 +3161,7 @@ def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name): with pytest.raises(InvalidOrderException): exc = InvalidOrderException("Did not find order") mocker.patch(f'{EXMS}.cancel_stoploss_order', side_effect=exc) - mocker.patch('freqtrade.exchange.gate.Gate.cancel_stoploss_order', side_effect=exc) + mocker.patch(f'{mock_prefix}.cancel_stoploss_order', side_effect=exc) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=123) @@ -3223,8 +3236,14 @@ def test_fetch_stoploss_order(default_conf, mocker, exchange_name): api_mock = MagicMock() api_mock.fetch_order = MagicMock(return_value={'id': '123', 'symbol': 'TKN/BTC'}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - assert exchange.fetch_stoploss_order('X', 'TKN/BTC') == {'id': '123', 'symbol': 'TKN/BTC'} + res = {'id': '123', 'symbol': 'TKN/BTC'} + if exchange_name == 'okx': + res = {'id': '123', 'symbol': 'TKN/BTC', 'type': 'stoploss'} + assert exchange.fetch_stoploss_order('X', 'TKN/BTC') == res + if exchange_name == 'okx': + # Tested separately. + return with pytest.raises(InvalidOrderException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) @@ -3544,6 +3563,8 @@ def test_get_markets_error(default_conf, mocker): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_ohlcv_candle_limit(default_conf, mocker, exchange_name): + if exchange_name == 'okx': + pytest.skip("Tested separately for okx") exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) timeframes = ('1m', '5m', '1h') expected = exchange._ft_has['ohlcv_candle_limit'] diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index e8f059118..736c630e0 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -618,3 +618,70 @@ def test__get_stop_params_okx(mocker, default_conf): assert params['tdMode'] == 'isolated' assert params['posSide'] == 'net' + + +def test_fetch_orders_okx(default_conf, mocker, limit_order): + + api_mock = MagicMock() + api_mock.fetch_orders = MagicMock(return_value=[ + limit_order['buy'], + limit_order['sell'], + ]) + api_mock.fetch_open_orders = MagicMock(return_value=[limit_order['buy']]) + api_mock.fetch_closed_orders = MagicMock(return_value=[limit_order['buy']]) + + mocker.patch(f'{EXMS}.exchange_has', return_value=True) + start_time = datetime.now(timezone.utc) - timedelta(days=20) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='okx') + # Not available in dry-run + assert exchange.fetch_orders('mocked', start_time) == [] + assert api_mock.fetch_orders.call_count == 0 + default_conf['dry_run'] = False + + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='okx') + + def has_resp(_, endpoint): + if endpoint == 'fetchOrders': + return False + if endpoint == 'fetchClosedOrders': + return True + if endpoint == 'fetchOpenOrders': + return True + + mocker.patch(f'{EXMS}.exchange_has', has_resp) + + history_params = {'method': 'privateGetTradeOrdersHistoryArchive'} + + # happy path without fetchOrders + exchange.fetch_orders('mocked', start_time) + assert api_mock.fetch_orders.call_count == 0 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 2 + assert 'params' not in api_mock.fetch_closed_orders.call_args_list[0][1] + assert api_mock.fetch_closed_orders.call_args_list[1][1]['params'] == history_params + + api_mock.fetch_open_orders.reset_mock() + api_mock.fetch_closed_orders.reset_mock() + + # regular closed_orders endpoint only has history for 7 days. + exchange.fetch_orders('mocked', datetime.now(timezone.utc) - timedelta(days=6)) + assert api_mock.fetch_orders.call_count == 0 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 1 + assert 'params' not in api_mock.fetch_closed_orders.call_args_list[0][1] + + mocker.patch(f'{EXMS}.exchange_has', return_value=True) + + # Unhappy path - first fetch-orders call fails. + api_mock.fetch_orders = MagicMock(side_effect=ccxt.NotSupported()) + api_mock.fetch_open_orders.reset_mock() + api_mock.fetch_closed_orders.reset_mock() + + exchange.fetch_orders('mocked', start_time) + + assert api_mock.fetch_orders.call_count == 1 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 2 + assert 'params' not in api_mock.fetch_closed_orders.call_args_list[0][1] + assert api_mock.fetch_closed_orders.call_args_list[1][1]['params'] == history_params diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index 8ab2c75da..ca4749747 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -1,7 +1,9 @@ import shutil from pathlib import Path +from unittest.mock import patch +import pandas as pd import pytest from freqtrade.configuration import TimeRange @@ -135,3 +137,111 @@ def test_get_timerange_from_backtesting_live_df_pred_not_found(mocker, freqai_co match=r'Historic predictions not found.*' ): freqai.dd.get_timerange_from_live_historic_predictions() + + +def test_set_initial_return_values(mocker, freqai_conf): + """ + Simple test of the set initial return values that ensures + we are concatening and ffilling values properly. + """ + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + # Setup + pair = "BTC/USD" + end_x = "2023-08-31" + start_x_plus_1 = "2023-08-30" + end_x_plus_5 = "2023-09-03" + + historic_data = { + 'date_pred': pd.date_range(end=end_x, periods=5), + 'value': range(1, 6) + } + new_data = { + 'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5), + 'value': range(6, 11) + } + + freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data) + + new_pred_df = pd.DataFrame(new_data) + dataframe = pd.DataFrame(new_data) + + # Action + with patch('logging.Logger.warning') as mock_logger_warning: + freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe) + + # Assertions + hist_pred_df = freqai.dd.historic_predictions[pair] + model_return_df = freqai.dd.model_return_values[pair] + + assert (hist_pred_df['date_pred'].iloc[-1] == + pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1)) + assert 'date_pred' in hist_pred_df.columns + assert hist_pred_df.shape[0] == 7 # Total rows: 5 from historic and 2 new zeros + + # compare values in model_return_df with hist_pred_df + assert (model_return_df["value"].values == + hist_pred_df.tail(len(dataframe))["value"].values).all() + assert model_return_df.shape[0] == len(dataframe) + + # Ensure logger error is not called + mock_logger_warning.assert_not_called() + + +def test_set_initial_return_values_warning(mocker, freqai_conf): + """ + Simple test of set_initial_return_values that hits the warning + associated with leaving a FreqAI bot offline so long that the + exchange candles have no common date with the historic predictions + """ + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + # Setup + pair = "BTC/USD" + end_x = "2023-08-31" + start_x_plus_1 = "2023-09-01" + end_x_plus_5 = "2023-09-05" + + historic_data = { + 'date_pred': pd.date_range(end=end_x, periods=5), + 'value': range(1, 6) + } + new_data = { + 'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5), + 'value': range(6, 11) + } + + freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data) + + new_pred_df = pd.DataFrame(new_data) + dataframe = pd.DataFrame(new_data) + + # Action + with patch('logging.Logger.warning') as mock_logger_warning: + freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe) + + # Assertions + hist_pred_df = freqai.dd.historic_predictions[pair] + model_return_df = freqai.dd.model_return_values[pair] + + assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1) + assert 'date_pred' in hist_pred_df.columns + assert hist_pred_df.shape[0] == 9 # Total rows: 5 from historic and 4 new zeros + + # compare values in model_return_df with hist_pred_df + assert (model_return_df["value"].values == hist_pred_df.tail( + len(dataframe))["value"].values).all() + assert model_return_df.shape[0] == len(dataframe) + + # Ensure logger error is not called + mock_logger_warning.assert_called() diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 0f0057d1e..396d60c18 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -600,7 +600,9 @@ def test_calc_open_close_trade_price( @pytest.mark.usefixtures("init_persistence") -def test_trade_close(fee): +def test_trade_close(fee, time_machine): + time_machine.move_to("2022-09-01 05:00:00 +00:00", tick=False) + trade = Trade( pair='ADA/USDT', stake_amount=60.0, @@ -609,7 +611,7 @@ def test_trade_close(fee): is_open=True, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=10), + open_date=dt_now() - timedelta(minutes=10), interest_rate=0.0005, exchange='binance', trading_mode=margin, @@ -628,6 +630,7 @@ def test_trade_close(fee): status="closed", order_type="limit", side=trade.entry_side, + order_filled_date=trade.open_date, )) trade.orders.append(Order( ft_order_side=trade.exit_side, @@ -642,6 +645,7 @@ def test_trade_close(fee): status="closed", order_type="limit", side=trade.exit_side, + order_filled_date=dt_now(), )) assert trade.close_profit is None assert trade.close_date is None @@ -650,14 +654,15 @@ def test_trade_close(fee): assert trade.is_open is False assert pytest.approx(trade.close_profit) == 0.094513715 assert trade.close_date is not None + assert trade.close_date_utc == dt_now() - new_date = datetime(2020, 2, 2, 15, 6, 1), - assert trade.close_date != new_date + new_date = dt_now() + timedelta(minutes=5) + assert trade.close_date_utc != new_date # Close should NOT update close_date if the trade has been closed already assert trade.is_open is False trade.close_date = new_date trade.close(2.2) - assert trade.close_date == new_date + assert trade.close_date_utc == new_date @pytest.mark.usefixtures("init_persistence") @@ -1380,6 +1385,7 @@ def test_to_json(fee): precision_mode=1, amount_precision=8.0, price_precision=7.0, + contract_size=1, ) result = trade.to_json() assert isinstance(result, dict) @@ -1445,6 +1451,7 @@ def test_to_json(fee): 'amount_precision': 8.0, 'price_precision': 7.0, 'precision_mode': 1, + 'contract_size': 1, 'orders': [], 'has_open_orders': False, } @@ -1466,6 +1473,7 @@ def test_to_json(fee): precision_mode=2, amount_precision=7.0, price_precision=8.0, + contract_size=1 ) result = trade.to_json() assert isinstance(result, dict) @@ -1531,6 +1539,7 @@ def test_to_json(fee): 'amount_precision': 7.0, 'price_precision': 8.0, 'precision_mode': 2, + 'contract_size': 1, 'orders': [], 'has_open_orders': False, } @@ -1923,11 +1932,15 @@ def test_get_best_pair_lev(fee): @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize('is_short', [True, False]) -def test_get_exit_order_count(fee, is_short): +def test_get_canceled_exit_order_count(fee, is_short): create_mock_trades(fee, is_short=is_short) trade = Trade.get_trades([Trade.pair == 'ETC/BTC']).first() - assert trade.get_exit_order_count() == 1 + # No canceled order. + assert trade.get_canceled_exit_order_count() == 0 + + trade.orders[-1].status = 'canceled' + assert trade.get_canceled_exit_order_count() == 1 @pytest.mark.usefixtures("init_persistence") diff --git a/tests/persistence/test_trade_fromjson.py b/tests/persistence/test_trade_fromjson.py index 24522e744..24a693c75 100644 --- a/tests/persistence/test_trade_fromjson.py +++ b/tests/persistence/test_trade_fromjson.py @@ -66,6 +66,10 @@ def test_trade_fromjson(): "is_short": false, "trading_mode": "spot", "funding_fees": 0.0, + "amount_precision": 1.0, + "price_precision": 3.0, + "precision_mode": 2, + "contract_size": 1.0, "open_order_id": null, "orders": [ { @@ -180,6 +184,9 @@ def test_trade_fromjson(): assert isinstance(trade.open_date, datetime) assert trade.exit_reason == 'no longer good' assert trade.realized_profit == 2.76315361 + assert trade.precision_mode == 2 + assert trade.amount_precision == 1.0 + assert trade.contract_size == 1.0 assert len(trade.orders) == 5 last_o = trade.orders[-1] diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 49700b7f4..b8eb51a91 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -90,6 +90,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'amount_precision': 8.0, 'price_precision': 8.0, 'precision_mode': 2, + 'contract_size': 1, 'has_open_orders': False, 'orders': [{ 'amount': 91.07468123, 'average': 1.098e-05, 'safe_price': 1.098e-05, @@ -263,7 +264,11 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert isnan(fiat_profit_sum) -def test__rpc_timeunit_profit(default_conf_usdt, ticker, fee, markets, mocker) -> None: +def test__rpc_timeunit_profit( + default_conf_usdt, ticker, fee, markets, mocker, time_machine) -> None: + + time_machine.move_to("2023-09-05 10:00:00 +00:00", tick=False) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( EXMS, diff --git a/tests/strategy/strats/strategy_test_v3_custom_entry_price.py b/tests/strategy/strats/strategy_test_v3_custom_entry_price.py index 872984156..607ff6e1e 100644 --- a/tests/strategy/strats/strategy_test_v3_custom_entry_price.py +++ b/tests/strategy/strats/strategy_test_v3_custom_entry_price.py @@ -6,6 +6,8 @@ from typing import Optional from pandas import DataFrame from strategy_test_v3 import StrategyTestV3 +from freqtrade.persistence import Trade + class StrategyTestV3CustomEntryPrice(StrategyTestV3): """ @@ -31,7 +33,8 @@ class StrategyTestV3CustomEntryPrice(StrategyTestV3): def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: return dataframe - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Optional[Trade], current_time: datetime, + proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: return self.new_entry_price diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 886012535..0886c9fca 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2991,6 +2991,8 @@ def test_manage_open_orders_exit_usercustom( is_short, open_trade_usdt, caplog ) -> None: default_conf_usdt["unfilledtimeout"] = {"entry": 1440, "exit": 1440, "exit_timeout_count": 1} + limit_sell_order_old['amount'] = open_trade_usdt.amount + limit_sell_order_old['remaining'] = open_trade_usdt.amount if is_short: limit_sell_order_old['side'] = 'buy' @@ -3052,7 +3054,7 @@ def test_manage_open_orders_exit_usercustom( # 2nd canceled trade - Fail execute exit caplog.clear() - mocker.patch('freqtrade.persistence.Trade.get_exit_order_count', return_value=1) + mocker.patch('freqtrade.persistence.Trade.get_canceled_exit_order_count', return_value=1) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit', side_effect=DependencyException) freqtrade.manage_open_orders() @@ -5658,6 +5660,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize("is_short", [False, True]) def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_short, caplog): + default_conf_usdt['dry_run'] = False freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mock_uts = mocker.spy(freqtrade, 'update_trade_state') @@ -5669,17 +5672,17 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor ]) trade = Trade( - pair='ETH/USDT', - fee_open=0.001, - fee_close=0.001, - open_rate=entry_order['price'], - open_date=dt_now(), - stake_amount=entry_order['cost'], - amount=entry_order['amount'], - exchange="binance", - is_short=is_short, - leverage=1, - ) + pair='ETH/USDT', + fee_open=0.001, + fee_close=0.001, + open_rate=entry_order['price'], + open_date=dt_now(), + stake_amount=entry_order['cost'], + amount=entry_order['amount'], + exchange="binance", + is_short=is_short, + leverage=1, + ) trade.orders.append(Order.parse_from_ccxt_object( entry_order, 'ADA/USDT', entry_side(is_short)) @@ -5687,7 +5690,8 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor Trade.session.add(trade) freqtrade.handle_onexchange_order(trade) assert log_has_re(r"Found previously unknown order .*", caplog) - assert mock_uts.call_count == 1 + # Update trade state is called twice, once for the known and once for the unknown order. + assert mock_uts.call_count == 2 assert mock_fo.call_count == 1 trade = Trade.session.scalars(select(Trade)).first() @@ -5697,6 +5701,77 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor assert trade.exit_reason == ExitType.SOLD_ON_EXCHANGE.value +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize("is_short", [False, True]) +def test_handle_onexchange_order_exit(mocker, default_conf_usdt, limit_order, is_short, caplog): + default_conf_usdt['dry_run'] = False + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + mock_uts = mocker.spy(freqtrade, 'update_trade_state') + + entry_order = limit_order[entry_side(is_short)] + add_entry_order = deepcopy(entry_order) + add_entry_order.update({ + 'id': '_partial_entry_id', + 'amount': add_entry_order['amount'] / 1.5, + 'cost': add_entry_order['cost'] / 1.5, + 'filled': add_entry_order['filled'] / 1.5, + }) + + exit_order_part = deepcopy(limit_order[exit_side(is_short)]) + exit_order_part.update({ + 'id': 'some_random_partial_id', + 'amount': exit_order_part['amount'] / 2, + 'cost': exit_order_part['cost'] / 2, + 'filled': exit_order_part['filled'] / 2, + }) + exit_order = limit_order[exit_side(is_short)] + + # Orders intentionally in the wrong sequence + mock_fo = mocker.patch(f'{EXMS}.fetch_orders', return_value=[ + entry_order, + exit_order_part, + exit_order, + add_entry_order, + ]) + + trade = Trade( + pair='ETH/USDT', + fee_open=0.001, + fee_close=0.001, + open_rate=entry_order['price'], + open_date=dt_now(), + stake_amount=entry_order['cost'], + amount=entry_order['amount'], + exchange="binance", + is_short=is_short, + leverage=1, + is_open=True, + ) + + trade.orders = [ + Order.parse_from_ccxt_object(entry_order, trade.pair, entry_side(is_short)), + Order.parse_from_ccxt_object(exit_order_part, trade.pair, exit_side(is_short)), + Order.parse_from_ccxt_object(add_entry_order, trade.pair, entry_side(is_short)), + Order.parse_from_ccxt_object(exit_order, trade.pair, exit_side(is_short)), + ] + trade.recalc_trade_from_orders() + Trade.session.add(trade) + Trade.commit() + + freqtrade.handle_onexchange_order(trade) + # assert log_has_re(r"Found previously unknown order .*", caplog) + # Update trade state is called three times, once for every order + assert mock_uts.call_count == 4 + assert mock_fo.call_count == 1 + + trade = Trade.session.scalars(select(Trade)).first() + + assert len(trade.orders) == 4 + assert trade.is_open is True + assert trade.exit_reason is None + assert trade.amount == 5.0 + + def test_get_valid_price(mocker, default_conf_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) diff --git a/tests/test_misc.py b/tests/test_misc.py index e94e299fd..7de1adbbc 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -121,6 +121,8 @@ def test_safe_value_fallback(): assert safe_value_fallback(dict1, 'keyNo', 'keyNo') is None assert safe_value_fallback(dict1, 'keyNo', 'keyNo', 55) == 55 + assert safe_value_fallback(dict1, 'keyNo', default_value=55) == 55 + assert safe_value_fallback(dict1, 'keyNo', None, default_value=55) == 55 def test_safe_value_fallback2():