diff --git a/docs/commands/backtesting.md b/docs/commands/backtesting.md index 7082fb362..925b29da6 100644 --- a/docs/commands/backtesting.md +++ b/docs/commands/backtesting.md @@ -10,6 +10,7 @@ usage: freqtrade backtesting [-h] [-v] [--no-color] [--logfile FILE] [-V] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [-p PAIRS [PAIRS ...]] [--eps] [--enable-protections] + [--enable-dynamic-pairlist] [--dry-run-wallet DRY_RUN_WALLET] [--timeframe-detail TIMEFRAME_DETAIL] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] @@ -44,9 +45,14 @@ options: Allow buying the same pair multiple times (position stacking). --enable-protections, --enableprotections - Enable protections for backtesting.Will slow + Enable protections for backtesting. Will slow backtesting down by a considerable amount, but will include configured protections + --enable-dynamic-pairlist + Enables dynamic pairlist refreshes in backtesting. The + pairlist will be generated for each new candle if + you're using a pairlist handler that supports this + feature, for example, ShuffleFilter. --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET Starting balance, used for backtesting / hyperopt and dry-runs. diff --git a/docs/commands/hyperopt.md b/docs/commands/hyperopt.md index f1c8b28b5..933082b9d 100644 --- a/docs/commands/hyperopt.md +++ b/docs/commands/hyperopt.md @@ -44,7 +44,7 @@ options: Allow buying the same pair multiple times (position stacking). --enable-protections, --enableprotections - Enable protections for backtesting.Will slow + Enable protections for backtesting. Will slow backtesting down by a considerable amount, but will include configured protections --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET diff --git a/docs/commands/lookahead-analysis.md b/docs/commands/lookahead-analysis.md index 922e1f757..e94876308 100644 --- a/docs/commands/lookahead-analysis.md +++ b/docs/commands/lookahead-analysis.md @@ -11,6 +11,7 @@ usage: freqtrade lookahead-analysis [-h] [-v] [--no-color] [--logfile FILE] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [-p PAIRS [PAIRS ...]] [--enable-protections] + [--enable-dynamic-pairlist] [--dry-run-wallet DRY_RUN_WALLET] [--timeframe-detail TIMEFRAME_DETAIL] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] @@ -44,9 +45,14 @@ options: Limit command to these pairs. Pairs are space- separated. --enable-protections, --enableprotections - Enable protections for backtesting.Will slow + Enable protections for backtesting. Will slow backtesting down by a considerable amount, but will include configured protections + --enable-dynamic-pairlist + Enables dynamic pairlist refreshes in backtesting. The + pairlist will be generated for each new candle if + you're using a pairlist handler that supports this + feature, for example, ShuffleFilter. --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET Starting balance, used for backtesting / hyperopt and dry-runs. diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 8c3ddbcdb..6f60e6f86 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -49,6 +49,7 @@ ARGS_BACKTEST = [ *ARGS_COMMON_OPTIMIZE, "position_stacking", "enable_protections", + "enable_dynamic_pairlist", "dry_run_wallet", "timeframe_detail", "strategy_list", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index c00af1e97..9df2c3128 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -184,12 +184,20 @@ AVAILABLE_CLI_OPTIONS = { "enable_protections": Arg( "--enable-protections", "--enableprotections", - help="Enable protections for backtesting." + help="Enable protections for backtesting. " "Will slow backtesting down by a considerable amount, but will include " "configured protections", action="store_true", default=False, ), + "enable_dynamic_pairlist": Arg( + "--enable-dynamic-pairlist", + help="Enables dynamic pairlist refreshes in backtesting. " + "The pairlist will be generated for each new candle if you're using a " + "pairlist handler that supports this feature, for example, ShuffleFilter.", + action="store_true", + default=False, + ), "strategy_list": Arg( "--strategy-list", help="Provide a space-separated list of strategies to backtest. " diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index ec5fb62f5..a398ff970 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -259,7 +259,13 @@ class Configuration: self._args_to_config( config, argname="enable_protections", - logstring="Parameter --enable-protections detected, enabling Protections. ...", + logstring="Parameter --enable-protections detected, enabling Protections ...", + ) + + self._args_to_config( + config, + argname="enable_dynamic_pairlist", + logstring="Parameter --enable-dynamic-pairlist detected, enabling dynamic pairlist ...", ) if self.args.get("max_open_trades"): diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cbde89f80..30c0e745c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -211,6 +211,7 @@ class Backtesting: self._can_short = self.trading_mode != TradingMode.SPOT self._position_stacking: bool = self.config.get("position_stacking", False) self.enable_protections: bool = self.config.get("enable_protections", False) + self.dynamic_pairlist: bool = self.config.get("enable_dynamic_pairlist", False) migrate_data(config, self.exchange) self.init_backtest() @@ -1584,6 +1585,11 @@ class Backtesting: for current_time in self._time_generator(start_date, end_date): # Loop for each main candle. self.check_abort() + + if self.dynamic_pairlist and self.pairlists: + self.pairlists.refresh_pairlist() + pairs = self.pairlists.whitelist + # Reset open trade count for this candle # Critical to avoid exceeding max_open_trades in backtesting # when timeframe-detail is used and trades close within the opening candle. diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index be536e705..86d618160 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -93,6 +93,8 @@ class ShuffleFilter(IPairList): return pairlist_new # Shuffle is done inplace self._random.shuffle(pairlist) - self.__pairlist_cache[pairlist_bef] = pairlist + + if self._config.get("runmode") in (RunMode.LIVE, RunMode.DRY_RUN): + self.__pairlist_cache[pairlist_bef] = pairlist return pairlist diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index bff2105b3..94ba04235 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -7,6 +7,9 @@ Provides pair white list as it configured in config import logging from copy import deepcopy +from cachetools import LRUCache + +from freqtrade.enums import RunMode from freqtrade.exchange.exchange_types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting @@ -22,6 +25,8 @@ class StaticPairList(IPairList): super().__init__(*args, **kwargs) self._allow_inactive = self._pairlistconfig.get("allow_inactive", False) + # Pair cache - only used for optimize modes + self._bt_pair_cache: LRUCache = LRUCache(maxsize=1) @property def needstickers(self) -> bool: @@ -60,15 +65,23 @@ class StaticPairList(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: List of pairs """ - wl = self.verify_whitelist( - self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True - ) - if self._allow_inactive: - return wl - else: - # Avoid implicit filtering of "verify_whitelist" to keep - # proper warnings in the log - return self._whitelist_for_active_markets(wl) + pairlist = self._bt_pair_cache.get("pairlist") + + if not pairlist: + wl = self.verify_whitelist( + self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True + ) + if self._allow_inactive: + pairlist = wl + else: + # Avoid implicit filtering of "verify_whitelist" to keep + # proper warnings in the log + pairlist = self._whitelist_for_active_markets(wl) + + if self._config["runmode"] in (RunMode.BACKTEST, RunMode.HYPEROPT): + self._bt_pair_cache["pairlist"] = pairlist.copy() + + return pairlist def filter_pairlist(self, pairlist: list[str], tickers: Tickers) -> list[str]: """ diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index bc66d00c3..9056d842e 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -5,7 +5,7 @@ PairList manager class import logging from functools import partial -from cachetools import TTLCache, cached +from cachetools import LRUCache, TTLCache, cached from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider @@ -56,6 +56,7 @@ class PairListManager(LoggingMixin): ) self._check_backtest() + self._not_expiring_cache: LRUCache = LRUCache(maxsize=1) refresh_period = config.get("pairlist_refresh_period", 3600) LoggingMixin.__init__(self, logger, refresh_period) @@ -109,7 +110,15 @@ class PairListManager(LoggingMixin): @property def expanded_blacklist(self) -> list[str]: """The expanded blacklist (including wildcard expansion)""" - return expand_pairlist(self._blacklist, self._exchange.get_markets().keys()) + eblacklist = self._not_expiring_cache.get("eblacklist") + + if not eblacklist: + eblacklist = expand_pairlist(self._blacklist, self._exchange.get_markets().keys()) + + if self._config["runmode"] in (RunMode.BACKTEST, RunMode.HYPEROPT): + self._not_expiring_cache["eblacklist"] = eblacklist.copy() + + return eblacklist @property def name_list(self) -> list[str]: @@ -157,16 +166,17 @@ class PairListManager(LoggingMixin): :param logmethod: Function that'll be called, `logger.info` or `logger.warning`. :return: pairlist - blacklisted pairs """ - try: - blacklist = self.expanded_blacklist - except ValueError as err: - logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") - return [] - log_once = partial(self.log_once, logmethod=logmethod) - for pair in pairlist.copy(): - if pair in blacklist: - log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...") - pairlist.remove(pair) + if self._blacklist: + try: + blacklist = self.expanded_blacklist + except ValueError as err: + logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") + return [] + log_once = partial(self.log_once, logmethod=logmethod) + for pair in pairlist.copy(): + if pair in blacklist: + log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...") + pairlist.remove(pair) return pairlist def verify_whitelist( diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index adf88cb25..1c9373fb7 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -27,7 +27,7 @@ from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename, from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade, Trade from freqtrade.resolvers import StrategyResolver -from freqtrade.util.datetime_helpers import dt_utc +from freqtrade.util import dt_now, dt_utc from tests.conftest import ( CURRENT_TEST_STRATEGY, EXMS, @@ -2715,3 +2715,75 @@ def test_get_backtest_metadata_filename(): filename = "backtest_results_zip.zip" expected = Path("backtest_results_zip.meta.json") assert get_backtest_metadata_filename(filename) == expected + + +@pytest.mark.parametrize("dynamic_pairlist", [True, False]) +def test_time_pair_generator_refresh_pairlist(mocker, default_conf, dynamic_pairlist): + patch_exchange(mocker) + default_conf["enable_dynamic_pairlist"] = dynamic_pairlist + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + assert backtesting.dynamic_pairlist == dynamic_pairlist + + refresh_mock = mocker.patch( + "freqtrade.plugins.pairlistmanager.PairListManager.refresh_pairlist" + ) + + # Simulate 2 candles + start_date = datetime(2025, 1, 1, 0, 0, tzinfo=UTC) + end_date = start_date + timedelta(minutes=10) + pairs = default_conf["exchange"]["pair_whitelist"] + data = {pair: [] for pair in pairs} + + # Simulate backtest loop + list(backtesting.time_pair_generator(start_date, end_date, pairs, data)) + + if dynamic_pairlist: + assert refresh_mock.call_count == 2 + else: + assert refresh_mock.call_count == 0 + + +@pytest.mark.parametrize("dynamic_pairlist", [True, False]) +def test_time_pair_generator_open_trades_first(mocker, default_conf, dynamic_pairlist): + patch_exchange(mocker) + default_conf["enable_dynamic_pairlist"] = dynamic_pairlist + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + assert backtesting.dynamic_pairlist == dynamic_pairlist + + pairs = ["XRP/BTC", "LTC/BTC", "NEO/BTC", "ETH/BTC"] + + # Simulate open trades + trades = [ + LocalTrade(pair="XRP/BTC", open_date=dt_now(), amount=1, open_rate=1), + LocalTrade(pair="NEO/BTC", open_date=dt_now(), amount=1, open_rate=1), + ] + LocalTrade.bt_trades_open = trades + LocalTrade.bt_trades_open_pp = { + "XRP/BTC": [trades[0]], + "NEO/BTC": [trades[1]], + "LTC/BTC": [], + "ETH/BTC": [], + } + + start_date = datetime(2025, 1, 1, 0, 0, tzinfo=UTC) + end_date = start_date + timedelta(minutes=5) + dummy_row = (end_date, 1.0, 1.1, 0.9, 1.0, 0, 0, 0, 0, None, None) + data = {pair: [dummy_row] for pair in pairs} + + def mock_refresh(self): + # Simulate shuffle + self._whitelist = pairs[::-1] # ['ETH/BTC', 'NEO/BTC', 'LTC/BTC', 'XRP/BTC'] + + mocker.patch("freqtrade.plugins.pairlistmanager.PairListManager.refresh_pairlist", mock_refresh) + + processed_pairs = [] + for _, pair, _, _, _ in backtesting.time_pair_generator(start_date, end_date, pairs, data): + processed_pairs.append(pair) + + # Open trades first in both cases + if dynamic_pairlist: + assert processed_pairs == ["XRP/BTC", "NEO/BTC", "ETH/BTC", "LTC/BTC"] + else: + assert processed_pairs == ["XRP/BTC", "NEO/BTC", "LTC/BTC", "ETH/BTC"] diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index f12bc31ec..da4a3a780 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1274,27 +1274,37 @@ def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None: {"method": "StaticPairList"}, {"method": "ShuffleFilter", "seed": 43}, ] - whitelist_conf["runmode"] = "backtest" + whitelist_conf["runmode"] = RunMode.BACKTEST exchange = get_patched_exchange(mocker, whitelist_conf) plm = PairListManager(exchange, whitelist_conf) assert log_has("Backtesting mode detected, applying seed value: 43", caplog) + plm.refresh_pairlist() + pl1 = deepcopy(plm.whitelist) + plm.refresh_pairlist() + assert plm.whitelist != pl1 + assert set(plm.whitelist) == set(pl1) + + caplog.clear() + whitelist_conf["runmode"] = RunMode.DRY_RUN + plm = PairListManager(exchange, whitelist_conf) + assert not log_has("Backtesting mode detected, applying seed value: 42", caplog) + assert log_has("Live mode detected, not applying seed.", caplog) + with time_machine.travel("2021-09-01 05:01:00 +00:00") as t: plm.refresh_pairlist() pl1 = deepcopy(plm.whitelist) plm.refresh_pairlist() assert plm.whitelist == pl1 + target = plm._pairlist_handlers[1]._random + shuffle_mock = mocker.patch.object(target, "shuffle", wraps=target.shuffle) + t.shift(timedelta(minutes=10)) plm.refresh_pairlist() - assert plm.whitelist != pl1 - - caplog.clear() - whitelist_conf["runmode"] = RunMode.DRY_RUN - plm = PairListManager(exchange, whitelist_conf) - assert not log_has("Backtesting mode detected, applying seed value: 42", caplog) - assert log_has("Live mode detected, not applying seed.", caplog) + assert shuffle_mock.call_count == 1 + assert set(plm.whitelist) == set(pl1) @pytest.mark.usefixtures("init_persistence")