diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index d13fa6fca..6a4d796ea 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -5,6 +5,14 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple from pandas import DataFrame, concat +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, +) from freqtrade.configuration import TimeRange from freqtrade.constants import ( @@ -29,6 +37,7 @@ from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.util import dt_ts, format_ms_time from freqtrade.util.datetime_helpers import dt_now from freqtrade.util.migrations import migrate_data +from freqtrade.util.rich_progress import CustomProgress logger = logging.getLogger(__name__) @@ -155,11 +164,9 @@ def refresh_data( :param candle_type: Any of the enum CandleType (must match trading mode!) """ data_handler = get_datahandler(datadir, data_format) - for idx, pair in enumerate(pairs): - process = f"{idx}/{len(pairs)}" + for pair in pairs: _download_pair_history( pair=pair, - process=process, timeframe=timeframe, datadir=datadir, timerange=timerange, @@ -223,7 +230,6 @@ def _download_pair_history( datadir: Path, exchange: Exchange, timeframe: str = "5m", - process: str = "", new_pairs_days: int = 30, data_handler: Optional[IDataHandler] = None, timerange: Optional[TimeRange] = None, @@ -261,7 +267,7 @@ def _download_pair_history( ) logger.info( - f'({process}) - Download history data for "{pair}", {timeframe}, ' + f'Download history data for "{pair}", {timeframe}, ' f"{candle_type} and store in {datadir}. " f'From {format_ms_time(since_ms) if since_ms else "start"} to ' f'{format_ms_time(until_ms) if until_ms else "now"}' @@ -345,53 +351,75 @@ def refresh_backtest_ohlcv_data( pairs_not_available = [] data_handler = get_datahandler(datadir, data_format) candle_type = CandleType.get_default(trading_mode) - process = "" - for idx, pair in enumerate(pairs, start=1): - if pair not in exchange.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - for timeframe in timeframes: - logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.") - process = f"{idx}/{len(pairs)}" - _download_pair_history( - pair=pair, - process=process, - datadir=datadir, - exchange=exchange, - timerange=timerange, - data_handler=data_handler, - timeframe=str(timeframe), - new_pairs_days=new_pairs_days, - candle_type=candle_type, - erase=erase, - prepend=prepend, - ) - if trading_mode == "futures": - # Predefined candletype (and timeframe) depending on exchange - # Downloads what is necessary to backtest based on futures data. - tf_mark = exchange.get_option("mark_ohlcv_timeframe") - tf_funding_rate = exchange.get_option("funding_fee_timeframe") - - fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price")) - # All exchanges need FundingRate for futures trading. - # The timeframe is aligned to the mark-price timeframe. - combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark)) - for candle_type_f, tf in combs: - logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.") + with CustomProgress( + TextColumn("[progress.description]{task.description}"), + BarColumn(bar_width=None), + MofNCompleteColumn(), + TaskProgressColumn(), + "•", + TimeElapsedColumn(), + "•", + TimeRemainingColumn(), + expand=True, + ) as progress: + tf_length = len(timeframes) if trading_mode != "futures" else len(timeframes) + 2 + timeframe_task = progress.add_task("Timeframe", total=tf_length) + pair_task = progress.add_task("Downloading data...", total=len(pairs)) + + for pair in pairs: + progress.update(pair_task, description=f"Downloading {pair}") + progress.update(timeframe_task, completed=0) + + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for timeframe in timeframes: + progress.update(timeframe_task, description=f"Timeframe {timeframe}") + logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.") _download_pair_history( pair=pair, - process=process, datadir=datadir, exchange=exchange, timerange=timerange, data_handler=data_handler, - timeframe=str(tf), + timeframe=str(timeframe), new_pairs_days=new_pairs_days, - candle_type=candle_type_f, + candle_type=candle_type, erase=erase, prepend=prepend, ) + progress.update(timeframe_task, advance=1) + if trading_mode == "futures": + # Predefined candletype (and timeframe) depending on exchange + # Downloads what is necessary to backtest based on futures data. + tf_mark = exchange.get_option("mark_ohlcv_timeframe") + tf_funding_rate = exchange.get_option("funding_fee_timeframe") + + fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price")) + # All exchanges need FundingRate for futures trading. + # The timeframe is aligned to the mark-price timeframe. + combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark)) + for candle_type_f, tf in combs: + logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.") + _download_pair_history( + pair=pair, + datadir=datadir, + exchange=exchange, + timerange=timerange, + data_handler=data_handler, + timeframe=str(tf), + new_pairs_days=new_pairs_days, + candle_type=candle_type_f, + erase=erase, + prepend=prepend, + ) + progress.update( + timeframe_task, advance=1, description=f"Timeframe {candle_type_f}, {tf}" + ) + + progress.update(pair_task, advance=1) + progress.update(timeframe_task, description="Timeframe") return pairs_not_available diff --git a/tests/data/test_history.py b/tests/data/test_history.py index e3f92b1f9..e613bc589 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -151,9 +151,7 @@ def test_load_data_with_new_pair_1min( ) load_pair_history(datadir=tmp_path, timeframe="1m", pair="MEME/BTC", candle_type=candle_type) assert file.is_file() - assert log_has_re( - r'\(0/1\) - Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog - ) + assert log_has_re(r'Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog) def test_testdata_path(testdatadir) -> None: