lychiyu/freqtrade_origin
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2024-11-10 02:12:01 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Correct calculation for percent calculation and use tickers

Browse Source
This commit is contained in:
jainanuj94 2024-07-25 23:33:28 +05:30
parent 1b81de01b4
commit dad4f30597
7 changed files with 195 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/constants.py
View File

@ -42,7 +42,7 @@ HYPEROPT_LOSS_BUILTIN = [
AVAILABLE_PAIRLISTS = [
"StaticPairList",
"VolumePairList",
"PercentVolumeChangePairList",
"PercentChangePairList",
"ProducerPairList",
"RemotePairList",
"MarketCapPairList",

1
freqtrade/exchange/exchange.py
View File

@ -121,6 +121,7 @@ class Exchange:
# Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency
"ohlcv_volume_currency": "base", # "base" or "quote"
"tickers_have_quoteVolume": True,
"tickers_have_percentage": True,
"tickers_have_bid_ask": True, # bid / ask empty for fetch_tickers
"tickers_have_price": True,
"trades_pagination": "time", # Possible are "time" or "id"

1
freqtrade/exchange/types.py
View File

@ -12,6 +12,7 @@ class Ticker(TypedDict):
last: Optional[float]
quoteVolume: Optional[float]
baseVolume: Optional[float]
percentage: Optional[float]
# Several more - only listing required.

203
freqtrade/plugins/pairlist/PercentVolumeChangePairList.py → freqtrade/plugins/pairlist/PercentChangePairList.py
View File

@ -8,24 +8,24 @@ defined period
import logging
from datetime import timedelta
from typing import Any, Dict, List, Literal
from typing import Any, Dict, List, Literal, Optional
from cachetools import TTLCache
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
from freqtrade.exchange.types import Tickers
from freqtrade.exchange.types import Ticker, Tickers
from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
from freqtrade.util import dt_now, format_ms_time
logger = logging.getLogger(__name__)
SORT_VALUES = ["rolling_volume_change"]
SORT_VALUES = ["percentage"]
class PercentVolumeChangePairList(IPairList):
class PercentChangePairList(IPairList):
is_pairlist_generator = True
supports_backtesting = SupportsBacktesting.NO
@ -50,6 +50,7 @@ class PercentVolumeChangePairList(IPairList):
self._lookback_days = self._pairlistconfig.get("lookback_days", 0)
self._lookback_timeframe = self._pairlistconfig.get("lookback_timeframe", "1d")
self._lookback_period = self._pairlistconfig.get("lookback_period", 0)
self._sort_direction: Optional[str] = self._pairlistconfig.get("sort_direction", "desc")
self._def_candletype = self._config["candle_type_def"]
if (self._lookback_days > 0) & (self._lookback_period > 0):
@ -80,11 +81,11 @@ class PercentVolumeChangePairList(IPairList):
if not self._use_range and not (
self._exchange.exchange_has("fetchTickers")
and self._exchange.get_option("tickers_have_change")
and self._exchange.get_option("tickers_have_percentage")
):
raise OperationalException(
"Exchange does not support dynamic whitelist in this configuration. "
"Please edit your config and either remove PercentVolumeChangePairList, "
"Please edit your config and either remove PercentChangePairList, "
"or switch to using candles. and restart the bot."
)
@ -94,9 +95,7 @@ class PercentVolumeChangePairList(IPairList):
candle_limit = self._exchange.ohlcv_candle_limit(
self._lookback_timeframe, self._config["candle_type_def"]
)
if self._lookback_period < 4:
raise OperationalException("ChangeFilter requires lookback_period to be >= 4")
self.log_once(f"Candle limit is {candle_limit}", logger.info)
if self._lookback_period > candle_limit:
raise OperationalException(
"ChangeFilter requires lookback_period to not "
@ -119,14 +118,11 @@ class PercentVolumeChangePairList(IPairList):
"""
Short whitelist method description - used for startup-messages
"""
return (
f"{self.name} - top {self._pairlistconfig['number_assets']} percent "
f"volume change pairs."
)
return f"{self.name} - top {self._pairlistconfig['number_assets']} percent change pairs."
@staticmethod
def description() -> str:
return "Provides dynamic pair list based on percentage volume change."
return "Provides dynamic pair list based on percentage change."
@staticmethod
def available_parameters() -> Dict[str, PairlistParameter]:
@ -156,12 +152,14 @@ class PercentVolumeChangePairList(IPairList):
"description": "Maximum value",
"help": "Maximum value to use for filtering the pairlist.",
},
"refresh_period": {
"type": "number",
"default": 1800,
"description": "Refresh period",
"help": "Refresh period in seconds",
"sort_direction": {
"type": "option",
"default": "desc",
"options": ["", "asc", "desc"],
"description": "Sort pairlist",
"help": "Sort Pairlist ascending or descending by rate of change.",
},
**IPairList.refresh_period_parameter(),
"lookback_days": {
"type": "number",
"default": 0,
@ -233,83 +231,24 @@ class PercentVolumeChangePairList(IPairList):
:param tickers: Tickers (from exchange.get_tickers). May be cached.
:return: new whitelist
"""
self.log_once(f"Filter ticker is self use range {pairlist}", logger.warning)
filtered_tickers: List[Dict[str, Any]] = [{"symbol": k} for k in pairlist]
if self._use_range:
filtered_tickers: List[Dict[str, Any]] = [{"symbol": k} for k in pairlist]
# get lookback period in ms, for exchange ohlcv fetch
since_ms = (
int(
timeframe_to_prev_date(
self._lookback_timeframe,
dt_now()
+ timedelta(
minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min
),
).timestamp()
)
* 1000
)
to_ms = (
int(
timeframe_to_prev_date(
self._lookback_timeframe, dt_now() - timedelta(minutes=self._tf_in_min)
).timestamp()
)
* 1000
)
# todo: utc date output for starting date
self.log_once(
f"Using change range of {self._lookback_period} candles, timeframe: "
f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
f"till {format_ms_time(to_ms)}",
logger.info,
)
needed_pairs: ListPairsWithTimeframes = [
(p, self._lookback_timeframe, self._def_candletype)
for p in [s["symbol"] for s in filtered_tickers]
if p not in self._pair_cache
]
candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms)
for i, p in enumerate(filtered_tickers):
pair_candles = (
candles[(p["symbol"], self._lookback_timeframe, self._def_candletype)]
if (p["symbol"], self._lookback_timeframe, self._def_candletype) in candles
else None
)
# in case of candle data calculate typical price and change for candle
if pair_candles is not None and not pair_candles.empty:
pair_candles["rolling_volume_sum"] = (
pair_candles["volume"].rolling(window=self._lookback_period).sum()
)
pair_candles["rolling_volume_change"] = (
pair_candles["rolling_volume_sum"].pct_change() * 100
)
# ensure that a rolling sum over the lookback_period is built
# if pair_candles contains more candles than lookback_period
rolling_volume_change = pair_candles["rolling_volume_change"].fillna(0).iloc[-1]
# replace change with a range change sum calculated above
filtered_tickers[i]["rolling_volume_change"] = rolling_volume_change
self.log_once(f"ticker {filtered_tickers[i]}", logger.info)
else:
filtered_tickers[i]["rolling_volume_change"] = 0
# calculating using lookback_period
self.fetch_percent_change_from_lookback_period(filtered_tickers)
else:
filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
# Fetching 24h change by default from supported exchange tickers
self.fetch_percent_change_from_tickers(filtered_tickers, tickers)
filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] > self._min_value]
if self._max_value is not None:
filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] < self._max_value]
sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key])
sorted_tickers = sorted(
filtered_tickers,
reverse=self._sort_direction == "desc",
key=lambda t: t[self._sort_key],
)
self.log_once(f"Sorted Tickers {sorted_tickers}", logger.info)
# Validate whitelist to only have active market pairs
pairs = self._whitelist_for_active_markets([s["symbol"] for s in sorted_tickers])
pairs = self.verify_blacklist(pairs, logmethod=logger.info)
@ -317,3 +256,91 @@ class PercentVolumeChangePairList(IPairList):
pairs = pairs[: self._number_pairs]
return pairs
def fetch_candles_for_lookback_period(self, filtered_tickers):
since_ms = (
int(
timeframe_to_prev_date(
self._lookback_timeframe,
dt_now()
+ timedelta(
minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min
),
).timestamp()
)
* 1000
)
to_ms = (
int(
timeframe_to_prev_date(
self._lookback_timeframe, dt_now() - timedelta(minutes=self._tf_in_min)
).timestamp()
)
* 1000
)
# todo: utc date output for starting date
self.log_once(
f"Using change range of {self._lookback_period} candles, timeframe: "
f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
f"till {format_ms_time(to_ms)}",
logger.info,
)
needed_pairs: ListPairsWithTimeframes = [
(p, self._lookback_timeframe, self._def_candletype)
for p in [s["symbol"] for s in filtered_tickers]
if p not in self._pair_cache
]
candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms)
return candles
def fetch_percent_change_from_lookback_period(self, filtered_tickers):
# get lookback period in ms, for exchange ohlcv fetch
candles = self.fetch_candles_for_lookback_period(filtered_tickers)
for i, p in enumerate(filtered_tickers):
pair_candles = (
candles[(p["symbol"], self._lookback_timeframe, self._def_candletype)]
if (p["symbol"], self._lookback_timeframe, self._def_candletype) in candles
else None
)
# in case of candle data calculate typical price and change for candle
if pair_candles is not None and not pair_candles.empty:
current_close = pair_candles["close"].iloc[-1]
previous_close = pair_candles["close"].shift(self._lookback_period).iloc[-1]
pct_change = (
((current_close - previous_close) / previous_close) if previous_close > 0 else 0
)
# replace change with a range change sum calculated above
filtered_tickers[i]["percentage"] = pct_change
self.log_once(f"Tickers: {filtered_tickers}", logger.info)
else:
filtered_tickers[i]["percentage"] = 0
def fetch_percent_change_from_tickers(self, filtered_tickers, tickers):
for i, p in enumerate(filtered_tickers):
# Filter out assets
if not self._validate_pair(
p["symbol"], tickers[p["symbol"]] if p["symbol"] in tickers else None
):
filtered_tickers.remove(p)
else:
filtered_tickers[i]["percentage"] = tickers[p["symbol"]]["percentage"]
def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool:
"""
Check if one price-step (pip) is > than a certain barrier.
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.fetch_ticker
:return: True if the pair can stay, false if it should be removed
"""
if not ticker or "percentage" not in ticker or ticker["percentage"] is None:
self.log_once(
f"Removed {pair} from whitelist, because "
"ticker['percentage'] is empty (Usually no trade in the last 24h).",
logger.info,
)
return False
return True

2
tests/conftest.py
View File

@ -2187,7 +2187,7 @@ def tickers():
"first": None,
"last": 530.21,
"change": 0.558,
"percentage": None,
"percentage": 2.349,
"average": None,
"baseVolume": 72300.0659,
"quoteVolume": 37670097.3022171,

6
tests/plugins/test_pairlist.py
View File

@ -31,11 +31,11 @@ from tests.conftest import (
)
# Exclude RemotePairList and PercentVolumeChangePairList from tests.
# Exclude RemotePairList and PercentVolumePairList from tests.
# They have mandatory parameters, and requires special handling,
# which happens in test_remotepairlist and test_percentvolumechangepairlist.
# which happens in test_remotepairlist and test_percentchangepairlist.
TESTABLE_PAIRLISTS = [
p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairList", "PercentVolumeChangePairList"]
p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairList", "PercentChangePairList"]
]

133
tests/plugins/test_percentvolumechangepairlist.py → tests/plugins/test_percentchangepairlist.py
View File

@ -7,7 +7,7 @@ import pytest
from freqtrade.data.converter import ohlcv_to_dataframe
from freqtrade.enums import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.PercentVolumeChangePairList import PercentVolumeChangePairList
from freqtrade.plugins.pairlist.PercentChangePairList import PercentChangePairList
from freqtrade.plugins.pairlistmanager import PairListManager
from tests.conftest import (
EXMS,
@ -33,9 +33,9 @@ def rpl_config(default_conf):
def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"method": "PercentChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"sort_key": "percentage",
"min_value": 0,
"refresh_period": 86400,
}
@ -44,7 +44,7 @@ def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config):
with pytest.raises(
OperationalException,
match=r"Exchange does not support dynamic whitelist in this configuration. "
r"Please edit your config and either remove PercentVolumeChangePairList, "
r"Please edit your config and either remove PercentChangePairList, "
r"or switch to using candles. and restart the bot.",
):
get_patched_freqtradebot(mocker, rpl_config)
@ -53,9 +53,9 @@ def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config):
def test_volume_change_pair_list_init_wrong_refresh_period(mocker, rpl_config):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"method": "PercentChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"sort_key": "percentage",
"min_value": 0,
"refresh_period": 1800,
"lookback_days": 4,
@ -71,12 +71,31 @@ def test_volume_change_pair_list_init_wrong_refresh_period(mocker, rpl_config):
get_patched_freqtradebot(mocker, rpl_config)
def test_volume_change_pair_list_init_invalid_sort_key(mocker, rpl_config):
rpl_config["pairlists"] = [
{
"method": "PercentChangePairList",
"number_assets": 2,
"sort_key": "wrong_key",
"min_value": 0,
"refresh_period": 86400,
"lookback_days": 1,
}
]
with pytest.raises(
OperationalException,
match=r"key wrong_key not in \['percentage'\]",
):
get_patched_freqtradebot(mocker, rpl_config)
def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"method": "PercentChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"sort_key": "percentage",
"min_value": 0,
"refresh_period": 86400,
"lookback_days": 3,
@ -95,41 +114,9 @@ def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"method": "PercentChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 86400,
"lookback_days": 3,
}
]
with pytest.raises(
OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4"
):
get_patched_freqtradebot(mocker, rpl_config)
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 86400,
"lookback_period": 3,
}
]
with pytest.raises(
OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4"
):
get_patched_freqtradebot(mocker, rpl_config)
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"sort_key": "percentage",
"min_value": 0,
"refresh_period": 86400,
"lookback_days": 1001,
@ -147,8 +134,8 @@ def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config):
def test_volume_change_pair_list_init_wrong_config(mocker, rpl_config):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"sort_key": "rolling_volume_change",
"method": "PercentChangePairList",
"sort_key": "percentage",
"min_value": 0,
"refresh_period": 86400,
}
@ -165,9 +152,9 @@ def test_volume_change_pair_list_init_wrong_config(mocker, rpl_config):
def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tickers, time_machine):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"method": "PercentChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"sort_key": "percentage",
"min_value": 0,
"refresh_period": 86400,
"lookback_days": 4,
@ -210,7 +197,7 @@ def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tic
)
),
("TKN/USDT", "1d", CandleType.SPOT): pd.DataFrame(
# Make sure always have highest rolling_volume_change
# Make sure always have highest percentage
{
"timestamp": [
"2024-07-01 00:00:00",
@ -234,24 +221,25 @@ def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tic
exchange = get_patched_exchange(mocker, rpl_config, exchange="binance")
pairlistmanager = PairListManager(exchange, rpl_config)
remote_pairlist = PercentVolumeChangePairList(
remote_pairlist = PercentChangePairList(
exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0
)
result = remote_pairlist.gen_pairlist(tickers)
assert len(result) == 2
assert result == ["TKN/USDT", "BTC/USDT"]
assert result == ["NEO/USDT", "TKN/USDT"]
def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_machine):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"method": "PercentChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"sort_key": "percentage",
"min_value": 0,
"refresh_period": 86400,
"sort_direction": "asc",
"lookback_days": 4,
}
]
@ -272,7 +260,7 @@ def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_mac
"open": [100, 102, 101, 103, 104, 105],
"high": [102, 103, 102, 104, 105, 106],
"low": [99, 101, 100, 102, 103, 104],
"close": [101, 102, 103, 104, 105, 106],
"close": [101, 102, 103, 104, 105, 105],
"volume": [1000, 1500, 2000, 2500, 3000, 3500],
}
),
@ -289,8 +277,8 @@ def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_mac
"open": [100, 102, 101, 103, 104, 105],
"high": [102, 103, 102, 104, 105, 106],
"low": [99, 101, 100, 102, 103, 104],
"close": [101, 102, 103, 104, 105, 106],
"volume": [1000, 1500, 2000, 2500, 3000, 3500],
"close": [101, 102, 103, 104, 105, 104],
"volume": [1000, 1500, 2000, 2500, 3000, 3400],
}
),
}
@ -299,22 +287,22 @@ def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_mac
exchange = get_patched_exchange(mocker, rpl_config, exchange="binance")
pairlistmanager = PairListManager(exchange, rpl_config)
remote_pairlist = PercentVolumeChangePairList(
remote_pairlist = PercentChangePairList(
exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0
)
result = remote_pairlist.filter_pairlist(rpl_config["exchange"]["pair_whitelist"], {})
assert len(result) == 2
assert result == ["ETH/USDT", "XRP/USDT"]
assert result == ["XRP/USDT", "ETH/USDT"]
def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_machine):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"method": "PercentChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"sort_key": "percentage",
"min_value": 0,
"max_value": 15,
"refresh_period": 86400,
@ -356,7 +344,7 @@ def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_ma
"open": [100, 102, 101, 103, 104, 105],
"high": [102, 103, 102, 104, 105, 106],
"low": [99, 101, 100, 102, 103, 104],
"close": [101, 102, 103, 104, 105, 106],
"close": [101, 102, 103, 104, 105, 101],
"volume": [1000, 1500, 2000, 2500, 3000, 3500],
}
),
@ -366,7 +354,7 @@ def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_ma
exchange = get_patched_exchange(mocker, rpl_config, exchange="binance")
pairlistmanager = PairListManager(exchange, rpl_config)
remote_pairlist = PercentVolumeChangePairList(
remote_pairlist = PercentChangePairList(
exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0
)
@ -374,3 +362,28 @@ def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_ma
assert len(result) == 1
assert result == ["ETH/USDT"]
def test_gen_pairlist_from_tickers(mocker, rpl_config, tickers):
rpl_config["pairlists"] = [
{
"method": "PercentChangePairList",
"number_assets": 2,
"sort_key": "percentage",
"min_value": 0,
}
]
mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True))
exchange = get_patched_exchange(mocker, rpl_config, exchange="binance")
pairlistmanager = PairListManager(exchange, rpl_config)
remote_pairlist = PercentChangePairList(
exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0
)
result = remote_pairlist.gen_pairlist(tickers.return_value)
assert len(result) == 1
assert result == ["ETH/USDT"]