lychiyu/freqtrade_origin
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2024-11-10 10:21:59 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #10420 from freqtrade/feat/dl_data_progress

Browse Source 
Download Data - progressbar
This commit is contained in:
Matthias 2024-07-14 08:03:51 +02:00 committed by GitHub
commit b9c2868dff
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 106 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
freqtrade/data/history/history_utils.py
View File

@ -26,8 +26,7 @@ from freqtrade.enums import CandleType, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
from freqtrade.util import dt_ts, format_ms_time
from freqtrade.util.datetime_helpers import dt_now
from freqtrade.util import dt_now, dt_ts, format_ms_time, get_progress_tracker
from freqtrade.util.migrations import migrate_data
@ -155,11 +154,9 @@ def refresh_data(
:param candle_type: Any of the enum CandleType (must match trading mode!)
"""
data_handler = get_datahandler(datadir, data_format)
for idx, pair in enumerate(pairs):
process = f"{idx}/{len(pairs)}"
for pair in pairs:
_download_pair_history(
pair=pair,
process=process,
timeframe=timeframe,
datadir=datadir,
timerange=timerange,
@ -223,7 +220,6 @@ def _download_pair_history(
datadir: Path,
exchange: Exchange,
timeframe: str = "5m",
process: str = "",
new_pairs_days: int = 30,
data_handler: Optional[IDataHandler] = None,
timerange: Optional[TimeRange] = None,
@ -261,7 +257,7 @@ def _download_pair_history(
)
logger.info(
f'({process}) - Download history data for "{pair}", {timeframe}, '
f'Download history data for "{pair}", {timeframe}, '
f"{candle_type} and store in {datadir}. "
f'From {format_ms_time(since_ms) if since_ms else "start"} to '
f'{format_ms_time(until_ms) if until_ms else "now"}'
@ -345,53 +341,65 @@ def refresh_backtest_ohlcv_data(
pairs_not_available = []
data_handler = get_datahandler(datadir, data_format)
candle_type = CandleType.get_default(trading_mode)
process = ""
for idx, pair in enumerate(pairs, start=1):
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
for timeframe in timeframes:
logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
process = f"{idx}/{len(pairs)}"
_download_pair_history(
pair=pair,
process=process,
datadir=datadir,
exchange=exchange,
timerange=timerange,
data_handler=data_handler,
timeframe=str(timeframe),
new_pairs_days=new_pairs_days,
candle_type=candle_type,
erase=erase,
prepend=prepend,
)
if trading_mode == "futures":
# Predefined candletype (and timeframe) depending on exchange
# Downloads what is necessary to backtest based on futures data.
tf_mark = exchange.get_option("mark_ohlcv_timeframe")
tf_funding_rate = exchange.get_option("funding_fee_timeframe")
with get_progress_tracker() as progress:
tf_length = len(timeframes) if trading_mode != "futures" else len(timeframes) + 2
timeframe_task = progress.add_task("Timeframe", total=tf_length)
pair_task = progress.add_task("Downloading data...", total=len(pairs))
fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price"))
# All exchanges need FundingRate for futures trading.
# The timeframe is aligned to the mark-price timeframe.
combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
for candle_type_f, tf in combs:
logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
for pair in pairs:
progress.update(pair_task, description=f"Downloading {pair}")
progress.update(timeframe_task, completed=0)
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
for timeframe in timeframes:
progress.update(timeframe_task, description=f"Timeframe {timeframe}")
logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
_download_pair_history(
pair=pair,
process=process,
datadir=datadir,
exchange=exchange,
timerange=timerange,
data_handler=data_handler,
timeframe=str(tf),
timeframe=str(timeframe),
new_pairs_days=new_pairs_days,
candle_type=candle_type_f,
candle_type=candle_type,
erase=erase,
prepend=prepend,
)
progress.update(timeframe_task, advance=1)
if trading_mode == "futures":
# Predefined candletype (and timeframe) depending on exchange
# Downloads what is necessary to backtest based on futures data.
tf_mark = exchange.get_option("mark_ohlcv_timeframe")
tf_funding_rate = exchange.get_option("funding_fee_timeframe")
fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price"))
# All exchanges need FundingRate for futures trading.
# The timeframe is aligned to the mark-price timeframe.
combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
for candle_type_f, tf in combs:
logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
_download_pair_history(
pair=pair,
datadir=datadir,
exchange=exchange,
timerange=timerange,
data_handler=data_handler,
timeframe=str(tf),
new_pairs_days=new_pairs_days,
candle_type=candle_type_f,
erase=erase,
prepend=prepend,
)
progress.update(
timeframe_task, advance=1, description=f"Timeframe {candle_type_f}, {tf}"
)
progress.update(pair_task, advance=1)
progress.update(timeframe_task, description="Timeframe")
return pairs_not_available
@ -501,25 +509,30 @@ def refresh_backtest_trades_data(
"""
pairs_not_available = []
data_handler = get_datahandler(datadir, data_format=data_format)
for pair in pairs:
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
with get_progress_tracker() as progress:
pair_task = progress.add_task("Downloading data...", total=len(pairs))
for pair in pairs:
progress.update(pair_task, description=f"Downloading trades [{pair}]")
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
if erase:
if data_handler.trades_purge(pair, trading_mode):
logger.info(f"Deleting existing data for pair {pair}.")
if erase:
if data_handler.trades_purge(pair, trading_mode):
logger.info(f"Deleting existing data for pair {pair}.")
logger.info(f"Downloading trades for pair {pair}.")
_download_trades_history(
exchange=exchange,
pair=pair,
new_pairs_days=new_pairs_days,
timerange=timerange,
data_handler=data_handler,
trading_mode=trading_mode,
)
progress.update(pair_task, advance=1)
logger.info(f"Downloading trades for pair {pair}.")
_download_trades_history(
exchange=exchange,
pair=pair,
new_pairs_days=new_pairs_days,
timerange=timerange,
data_handler=data_handler,
trading_mode=trading_mode,
)
return pairs_not_available

21
freqtrade/optimize/hyperopt.py
View File

@ -19,14 +19,6 @@ from joblib.externals import cloudpickle
from pandas import DataFrame
from rich.align import Align
from rich.console import Console
from rich.progress import (
BarColumn,
MofNCompleteColumn,
TaskProgressColumn,
TextColumn,
TimeElapsedColumn,
TimeRemainingColumn,
)
from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
from freqtrade.data.converter import trim_dataframes
@ -48,7 +40,7 @@ from freqtrade.optimize.hyperopt_tools import (
)
from freqtrade.optimize.optimize_reports import generate_strategy_stats
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver
from freqtrade.util import CustomProgress
from freqtrade.util import get_progress_tracker
# Suppress scikit-learn FutureWarnings from skopt
@ -634,16 +626,7 @@ class Hyperopt:
)
# Define progressbar
with CustomProgress(
TextColumn("[progress.description]{task.description}"),
BarColumn(bar_width=None),
MofNCompleteColumn(),
TaskProgressColumn(),
"•",
TimeElapsedColumn(),
"•",
TimeRemainingColumn(),
expand=True,
with get_progress_tracker(
console=console,
cust_objs=[Align.center(self._hyper_out.table)],
) as pbar:

1
freqtrade/util/__init__.py
View File

@ -15,6 +15,7 @@ from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value
from freqtrade.util.ft_precise import FtPrecise
from freqtrade.util.measure_time import MeasureTime
from freqtrade.util.periodic_cache import PeriodicCache
from freqtrade.util.progress_tracker import get_progress_tracker # noqa F401
from freqtrade.util.rich_progress import CustomProgress
from freqtrade.util.rich_tables import print_df_rich_table, print_rich_table
from freqtrade.util.template_renderer import render_template, render_template_with_fallback # noqa

28
freqtrade/util/progress_tracker.py Normal file
View File

@ -0,0 +1,28 @@
from rich.progress import (
BarColumn,
MofNCompleteColumn,
TaskProgressColumn,
TextColumn,
TimeElapsedColumn,
TimeRemainingColumn,
)
from freqtrade.util.rich_progress import CustomProgress
def get_progress_tracker(**kwargs):
"""
Get progress Bar with custom columns.
"""
return CustomProgress(
TextColumn("[progress.description]{task.description}"),
BarColumn(bar_width=None),
MofNCompleteColumn(),
TaskProgressColumn(),
"•",
TimeElapsedColumn(),
"•",
TimeRemainingColumn(),
expand=True,
**kwargs,
)

2
freqtrade/util/rich_progress.py
View File

@ -5,7 +5,7 @@ from rich.progress import Progress
class CustomProgress(Progress):
def __init__(self, *args, cust_objs, **kwargs) -> None:
def __init__(self, *args, cust_objs=[], **kwargs) -> None:
self._cust_objs = cust_objs
super().__init__(*args, **kwargs)

4
tests/data/test_history.py
View File

@ -151,9 +151,7 @@ def test_load_data_with_new_pair_1min(
)
load_pair_history(datadir=tmp_path, timeframe="1m", pair="MEME/BTC", candle_type=candle_type)
assert file.is_file()
assert log_has_re(
r'\(0/1\) - Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog
)
assert log_has_re(r'Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog)
def test_testdata_path(testdatadir) -> None: