From d12a7ff18b4f09af248e51d331855b6f7aa196fb Mon Sep 17 00:00:00 2001 From: hippocritical Date: Wed, 22 Mar 2023 12:32:39 +0100 Subject: [PATCH 01/55] freqtrades' merge broke my side, fixed it by porting it over to my develop branch, no changes with this commit logic-wise. --- freqtrade/commands/__init__.py | 3 +- freqtrade/commands/arguments.py | 23 +- freqtrade/commands/strategy_utils_commands.py | 45 ++++ .../backtest_lookahead_bias_checker.py | 241 ++++++++++++++++++ 4 files changed, 308 insertions(+), 4 deletions(-) create mode 100644 freqtrade/strategy/backtest_lookahead_bias_checker.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 66a9c995b..8add45241 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -22,6 +22,7 @@ from freqtrade.commands.optimize_commands import (start_backtesting, start_backt start_edge, start_hyperopt) from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit -from freqtrade.commands.strategy_utils_commands import start_strategy_update +from freqtrade.commands.strategy_utils_commands import (start_backtest_lookahead_bias_checker, + start_strategy_update) from freqtrade.commands.trade_commands import start_trading from freqtrade.commands.webserver_commands import start_webserver diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 47aa37fdf..d79216b21 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -116,8 +116,14 @@ NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] -ARGS_STRATEGY_UTILS = ["strategy_list", "strategy_path", "recursive_strategy_search"] +ARGS_STRATEGY_UPDATER = ARGS_COMMON_OPTIMIZE + ["strategy_list"] +ARGS_BACKTEST_LOOKAHEAD_BIAS_CHECKER = ARGS_BACKTEST + + +# + ["target_trades", "minimum_trades", +# "target_trades", "exportfilename"] +# will be added when the base version works. class Arguments: """ @@ -192,7 +198,8 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.commands import (start_analysis_entries_exits, start_backtesting, + from freqtrade.commands import (start_analysis_entries_exits, + start_backtest_lookahead_bias_checker, start_backtesting, start_backtesting_show, start_convert_data, start_convert_db, start_convert_trades, start_create_userdir, start_download_data, start_edge, @@ -450,4 +457,14 @@ class Arguments: 'files to the current version', parents=[_common_parser]) strategy_updater_cmd.set_defaults(func=start_strategy_update) - self._build_args(optionlist=ARGS_STRATEGY_UTILS, parser=strategy_updater_cmd) + self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd) + + # Add backtest lookahead bias checker subcommand + backtest_lookahead_bias_checker_cmd = \ + subparsers.add_parser('backtest_lookahead_bias_checker', + help="checks for potential look ahead bias", + parents=[_common_parser]) + backtest_lookahead_bias_checker_cmd.set_defaults(func=start_backtest_lookahead_bias_checker) + + self._build_args(optionlist=ARGS_BACKTEST_LOOKAHEAD_BIAS_CHECKER, + parser=backtest_lookahead_bias_checker_cmd) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index e579ec475..8bce9d4f9 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -7,6 +7,7 @@ from typing import Any, Dict from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.resolvers import StrategyResolver +from freqtrade.strategy.backtest_lookahead_bias_checker import backtest_lookahead_bias_checker from freqtrade.strategy.strategyupdater import StrategyUpdater @@ -53,3 +54,47 @@ def start_conversion(strategy_obj, config): instance_strategy_updater.start(config, strategy_obj) elapsed = time.perf_counter() - start print(f"Conversion of {Path(strategy_obj['location']).name} took {elapsed:.1f} seconds.") + + # except: + # pass + + +def start_backtest_lookahead_bias_checker(args: Dict[str, Any]) -> None: + """ + Start the backtest bias tester script + :param args: Cli args from Arguments() + :return: None + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + strategy_objs = StrategyResolver.search_all_objects( + config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) + + filtered_strategy_objs = [] + if 'strategy_list' in args and args['strategy_list'] is not None: + for args_strategy in args['strategy_list']: + for strategy_obj in strategy_objs: + if (strategy_obj['name'] == args_strategy + and strategy_obj not in filtered_strategy_objs): + filtered_strategy_objs.append(strategy_obj) + break + + for filtered_strategy_obj in filtered_strategy_objs: + initialize_single_lookahead_bias_checker(filtered_strategy_obj, config) + else: + processed_locations = set() + for strategy_obj in strategy_objs: + if strategy_obj['location'] not in processed_locations: + processed_locations.add(strategy_obj['location']) + initialize_single_lookahead_bias_checker(strategy_obj, config) + + +def initialize_single_lookahead_bias_checker(strategy_obj, config): + # try: + print(f"Bias test of {Path(strategy_obj['location']).name} started.") + instance_backtest_lookahead_bias_checker = backtest_lookahead_bias_checker() + start = time.perf_counter() + instance_backtest_lookahead_bias_checker.start(config, strategy_obj) + elapsed = time.perf_counter() - start + print(f"checking look ahead bias via backtests of {Path(strategy_obj['location']).name} " + f"took {elapsed:.1f} seconds.") diff --git a/freqtrade/strategy/backtest_lookahead_bias_checker.py b/freqtrade/strategy/backtest_lookahead_bias_checker.py new file mode 100644 index 000000000..288786c19 --- /dev/null +++ b/freqtrade/strategy/backtest_lookahead_bias_checker.py @@ -0,0 +1,241 @@ +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument +from copy import deepcopy +from datetime import datetime, timedelta, timezone + +import pandas + +from freqtrade.configuration import TimeRange +from freqtrade.data.history import get_timerange +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.optimize.backtesting import Backtesting + + +class backtest_lookahead_bias_checker: + class varHolder: + timerange: TimeRange + data: pandas.DataFrame + indicators: pandas.DataFrame + result: pandas.DataFrame + compared: pandas.DataFrame + from_dt: datetime + to_dt: datetime + compared_dt: datetime + + class analysis: + def __init__(self): + self.total_signals = 0 + self.false_entry_signals = 0 + self.false_exit_signals = 0 + self.false_indicators = [] + self.has_bias = False + + total_signals: int + false_entry_signals: int + false_exit_signals: int + + false_indicators: list + has_bias: bool + + def __init__(self): + self.strategy_obj + self.current_analysis + self.config + self.full_varHolder + self.entry_varholder + self.exit_varholder + self.backtesting + self.signals_to_check: int = 20 + self.current_analysis + self.full_varHolder.from_dt + self.full_varHolder.to_dt + + @staticmethod + def dt_to_timestamp(dt): + timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) + return timestamp + + def get_result(self, backtesting, processed): + min_date, max_date = get_timerange(processed) + + result = backtesting.backtest( + processed=deepcopy(processed), + start_date=min_date, + end_date=max_date + ) + return result + + # analyzes two data frames with processed indicators and shows differences between them. + def analyze_indicators(self, full_vars, cut_vars, current_pair): + # extract dataframes + cut_df = cut_vars.indicators[current_pair] + full_df = full_vars.indicators[current_pair] + + # cut longer dataframe to length of the shorter + full_df_cut = full_df[ + (full_df.date == cut_vars.compared_dt) + ].reset_index(drop=True) + cut_df_cut = cut_df[ + (cut_df.date == cut_vars.compared_dt) + ].reset_index(drop=True) + + # compare dataframes + if full_df_cut.shape[0] != 0: + if cut_df_cut.shape[0] != 0: + compare_df = full_df_cut.compare(cut_df_cut) + + # skippedColumns = ["date", "open", "high", "low", "close", "volume"] + for col_name, values in compare_df.items(): + col_idx = compare_df.columns.get_loc(col_name) + compare_df_row = compare_df.iloc[0] + # compare_df now is comprised of tuples with [1] having either 'self' or 'other' + if 'other' in col_name[1]: + continue + self_value = compare_df_row[col_idx] + other_value = compare_df_row[col_idx + 1] + other_value = compare_df_row[col_idx + 1] + + # output differences + if self_value != other_value: + + if not self.current_analysis.false_indicators.__contains__(col_name[0]): + self.current_analysis.false_indicators.append(col_name[0]) + print(f"=> found look ahead bias in indicator {col_name[0]}. " + + f"{str(self_value)} != {str(other_value)}") + # return compare_df + + def report_signal(self, result, column_name, checked_timestamp): + df = result['results'] + row_count = df[column_name].shape[0] + + if row_count == 0: + return False + else: + + df_cut = df[(df[column_name] == checked_timestamp)] + if df_cut[column_name].shape[0] == 0: + # print("did NOT find the same signal in column " + column_name + + # " at timestamp " + str(checked_timestamp)) + return False + else: + return True + return False + + def prepare_data(self, varholder, var_pairs): + self.config['timerange'] = \ + str(int(self.dt_to_timestamp(varholder.from_dt))) + "-" + \ + str(int(self.dt_to_timestamp(varholder.to_dt))) + self.backtesting = Backtesting(self.config) + self.backtesting._set_strategy(self.backtesting.strategylist[0]) + varholder.data, varholder.timerange = self.backtesting.load_bt_data() + varholder.indicators = self.backtesting.strategy.advise_all_indicators(varholder.data) + varholder.result = self.get_result(self.backtesting, varholder.indicators) + + def start(self, config, strategy_obj: dict) -> None: + self.strategy_obj = strategy_obj + self.config = config + self.current_analysis = backtest_lookahead_bias_checker.analysis() + + max_try_signals: int = 3 + found_signals: int = 0 + continue_with_strategy = True + + # first we need to get the necessary entry/exit signals + # so we start by 14 days and increase in 1 month steps + # until we have the desired trade amount. + for try_buysignals in range(max_try_signals): # range(3) = 0..2 + # re-initialize backtesting-variable + self.full_varHolder = backtest_lookahead_bias_checker.varHolder() + + # define datetimes in human readable format + self.full_varHolder.from_dt = datetime(2022, 9, 1) + self.full_varHolder.to_dt = datetime(2022, 9, 15) + timedelta(days=30 * try_buysignals) + + self.prepare_data(self.full_varHolder, self.config['pairs']) + + found_signals = self.full_varHolder.result['results'].shape[0] + 1 + if try_buysignals == max_try_signals - 1: + if found_signals < self.signals_to_check / 2: + print(f"... only found {str(int(found_signals / 2))} " + f"buy signals for {self.strategy_obj['name']}. " + f"Cancelling...") + continue_with_strategy = False + else: + print( + f"Found {str(found_signals)} buy signals. " + f"Going with max {str(self.signals_to_check)} " + f" buy signals in the full timerange from " + f"{str(self.full_varHolder.from_dt)} to {str(self.full_varHolder.to_dt)}") + break + elif found_signals < self.signals_to_check: + print( + f"Only found {str(found_signals)} buy signals in the full timerange from " + f"{str(self.full_varHolder.from_dt)} to " + f"{str(self.full_varHolder.to_dt)}. " + f"will increase timerange trying to get at least " + f"{str(self.signals_to_check)} signals.") + else: + print( + f"Found {str(found_signals)} buy signals, more than necessary. " + f"Reducing to {str(self.signals_to_check)} " + f"checked buy signals in the full timerange from " + f"{str(self.full_varHolder.from_dt)} to {str(self.full_varHolder.to_dt)}") + break + if not continue_with_strategy: + return + + for idx, result_row in self.full_varHolder.result['results'].iterrows(): + if self.current_analysis.total_signals == self.signals_to_check: + break + + # if force-sold, ignore this signal since here it will unconditionally exit. + if result_row.close_date == self.dt_to_timestamp(self.full_varHolder.to_dt): + continue + + self.current_analysis.total_signals += 1 + + self.entry_varholder = backtest_lookahead_bias_checker.varHolder() + self.exit_varholder = backtest_lookahead_bias_checker.varHolder() + + self.entry_varholder.from_dt = self.full_varHolder.from_dt # result_row['open_date'] + self.entry_varholder.compared_dt = result_row['open_date'] + + # to_dt needs +1 candle since it won't buy on the last candle + self.entry_varholder.to_dt = result_row['open_date'] + \ + timedelta(minutes=timeframe_to_minutes(self.config['timeframe']) * 2) + + self.prepare_data(self.entry_varholder, [result_row['pair']]) + + # --- + # print("analyzing the sell signal") + # to_dt needs +1 candle since it will always sell all trades on the last candle + self.exit_varholder.from_dt = self.full_varHolder.from_dt # result_row['open_date'] + self.exit_varholder.to_dt = \ + result_row['close_date'] + \ + timedelta(minutes=timeframe_to_minutes(self.config['timeframe'])) + self.exit_varholder.compared_dt = result_row['close_date'] + + self.prepare_data(self.exit_varholder, [result_row['pair']]) + + # register if buy signal is broken + if not self.report_signal( + self.entry_varholder.result, + "open_date", self.entry_varholder.compared_dt): + self.current_analysis.false_entry_signals += 1 + + # register if buy or sell signal is broken + if not self.report_signal(self.entry_varholder.result, + "open_date", self.entry_varholder.compared_dt) \ + or not self.report_signal(self.exit_varholder.result, + "close_date", self.exit_varholder.compared_dt): + self.current_analysis.false_exit_signals += 1 + + self.analyze_indicators(self.full_varHolder, self.entry_varholder, result_row['pair']) + self.analyze_indicators(self.full_varHolder, self.exit_varholder, result_row['pair']) + + if self.current_analysis.false_entry_signals > 0 or \ + self.current_analysis.false_exit_signals > 0 or \ + len(self.current_analysis.false_indicators) > 0: + print(" => " + self.strategy_obj['name'] + ": bias detected!") + self.current_analysis.has_bias = True + else: + print(self.strategy_obj['name'] + ": no bias detected") From 7bd55971dc8e1ab8a447c12952885c5975ddaae0 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Tue, 28 Mar 2023 22:20:00 +0200 Subject: [PATCH 02/55] strategy_updater: removed args_common_optimize for strategy-updater backtest_lookahead_bias_checker: added args and cli-options for minimum and target trade amounts fixed code according to best-practice coding requests of matthias (CamelCase etc) --- freqtrade/commands/arguments.py | 7 +- freqtrade/commands/cli_options.py | 14 ++ freqtrade/commands/strategy_utils_commands.py | 27 +- .../backtest_lookahead_bias_checker.py | 235 ++++++++---------- 4 files changed, 149 insertions(+), 134 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index d79216b21..6cb727eaf 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -116,9 +116,10 @@ NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] -ARGS_STRATEGY_UPDATER = ARGS_COMMON_OPTIMIZE + ["strategy_list"] +ARGS_STRATEGY_UPDATER = ["strategy_list"] -ARGS_BACKTEST_LOOKAHEAD_BIAS_CHECKER = ARGS_BACKTEST +ARGS_BACKTEST_LOOKAHEAD_BIAS_CHECKER = ARGS_BACKTEST + ["minimum_trade_amount", + "targeted_trade_amount"] # + ["target_trades", "minimum_trades", @@ -461,7 +462,7 @@ class Arguments: # Add backtest lookahead bias checker subcommand backtest_lookahead_bias_checker_cmd = \ - subparsers.add_parser('backtest_lookahead_bias_checker', + subparsers.add_parser('backtest-lookahead-bias-checker', help="checks for potential look ahead bias", parents=[_common_parser]) backtest_lookahead_bias_checker_cmd.set_defaults(func=start_backtest_lookahead_bias_checker) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index f1474ec69..5d2af934f 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -675,4 +675,18 @@ AVAILABLE_CLI_OPTIONS = { help='Run backtest with ready models.', action='store_true' ), + "minimum_trade_amount": Arg( + '--minimum-trade-amount', + help='set INT minimum trade amount', + type=check_int_positive, + metavar='INT', + default=10, + ), + "targeted_trade_amount": Arg( + '--targeted-trade-amount', + help='set INT targeted trade amount', + type=check_int_positive, + metavar='INT', + default=20, + ) } diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 8bce9d4f9..663ea571a 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -7,7 +7,7 @@ from typing import Any, Dict from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.resolvers import StrategyResolver -from freqtrade.strategy.backtest_lookahead_bias_checker import backtest_lookahead_bias_checker +from freqtrade.strategy.backtest_lookahead_bias_checker import BacktestLookaheadBiasChecker from freqtrade.strategy.strategyupdater import StrategyUpdater @@ -67,9 +67,16 @@ def start_backtest_lookahead_bias_checker(args: Dict[str, Any]) -> None: """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + if args['targeted_trade_amount'] < args['minimum_trade_amount']: + # add logic that tells the user to check the configuration + # since this combo doesn't make any sense. + pass + strategy_objs = StrategyResolver.search_all_objects( config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) + bias_checker_instances = [] + filtered_strategy_objs = [] if 'strategy_list' in args and args['strategy_list'] is not None: for args_strategy in args['strategy_list']: @@ -80,21 +87,29 @@ def start_backtest_lookahead_bias_checker(args: Dict[str, Any]) -> None: break for filtered_strategy_obj in filtered_strategy_objs: - initialize_single_lookahead_bias_checker(filtered_strategy_obj, config) + bias_checker_instances = initialize_single_lookahead_bias_checker( + filtered_strategy_obj, config, args) else: processed_locations = set() for strategy_obj in strategy_objs: if strategy_obj['location'] not in processed_locations: processed_locations.add(strategy_obj['location']) - initialize_single_lookahead_bias_checker(strategy_obj, config) + bias_checker_instances = initialize_single_lookahead_bias_checker( + strategy_obj, config, args) + create_result_list(bias_checker_instances) + + +def create_result_list(bias_checker_instances): + pass -def initialize_single_lookahead_bias_checker(strategy_obj, config): +def initialize_single_lookahead_bias_checker(strategy_obj, config, args): # try: print(f"Bias test of {Path(strategy_obj['location']).name} started.") - instance_backtest_lookahead_bias_checker = backtest_lookahead_bias_checker() + instance_backtest_lookahead_bias_checker = BacktestLookaheadBiasChecker() start = time.perf_counter() - instance_backtest_lookahead_bias_checker.start(config, strategy_obj) + current_instance = instance_backtest_lookahead_bias_checker.start(config, strategy_obj, args) elapsed = time.perf_counter() - start print(f"checking look ahead bias via backtests of {Path(strategy_obj['location']).name} " f"took {elapsed:.1f} seconds.") + return current_instance diff --git a/freqtrade/strategy/backtest_lookahead_bias_checker.py b/freqtrade/strategy/backtest_lookahead_bias_checker.py index 288786c19..c4a321a4a 100644 --- a/freqtrade/strategy/backtest_lookahead_bias_checker.py +++ b/freqtrade/strategy/backtest_lookahead_bias_checker.py @@ -1,4 +1,4 @@ -# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument +import copy from copy import deepcopy from datetime import datetime, timedelta, timezone @@ -10,8 +10,8 @@ from freqtrade.exchange import timeframe_to_minutes from freqtrade.optimize.backtesting import Backtesting -class backtest_lookahead_bias_checker: - class varHolder: +class BacktestLookaheadBiasChecker: + class VarHolder: timerange: TimeRange data: pandas.DataFrame indicators: pandas.DataFrame @@ -21,7 +21,7 @@ class backtest_lookahead_bias_checker: to_dt: datetime compared_dt: datetime - class analysis: + class Analysis: def __init__(self): self.total_signals = 0 self.false_entry_signals = 0 @@ -37,24 +37,24 @@ class backtest_lookahead_bias_checker: has_bias: bool def __init__(self): - self.strategy_obj - self.current_analysis - self.config - self.full_varHolder - self.entry_varholder - self.exit_varholder - self.backtesting - self.signals_to_check: int = 20 - self.current_analysis - self.full_varHolder.from_dt - self.full_varHolder.to_dt + self.strategy_obj = None + self.current_analysis = None + self.local_config = None + self.full_varHolder = None + self.entry_varHolder = None + self.exit_varHolder = None + self.backtesting = None + self.current_analysis = None + self.minimum_trade_amount = None + self.targeted_trade_amount = None @staticmethod def dt_to_timestamp(dt): timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) return timestamp - def get_result(self, backtesting, processed): + @staticmethod + def get_result(backtesting, processed): min_date, max_date = get_timerange(processed) result = backtesting.backtest( @@ -64,6 +64,24 @@ class backtest_lookahead_bias_checker: ) return result + @staticmethod + def report_signal(result, column_name, checked_timestamp): + df = result['results'] + row_count = df[column_name].shape[0] + + if row_count == 0: + return False + else: + + df_cut = df[(df[column_name] == checked_timestamp)] + if df_cut[column_name].shape[0] == 0: + # print("did NOT find the same signal in column " + column_name + + # " at timestamp " + str(checked_timestamp)) + return False + else: + return True + return False + # analyzes two data frames with processed indicators and shows differences between them. def analyze_indicators(self, full_vars, cut_vars, current_pair): # extract dataframes @@ -87,12 +105,11 @@ class backtest_lookahead_bias_checker: for col_name, values in compare_df.items(): col_idx = compare_df.columns.get_loc(col_name) compare_df_row = compare_df.iloc[0] - # compare_df now is comprised of tuples with [1] having either 'self' or 'other' + # compare_df now comprises tuples with [1] having either 'self' or 'other' if 'other' in col_name[1]: continue self_value = compare_df_row[col_idx] other_value = compare_df_row[col_idx + 1] - other_value = compare_df_row[col_idx + 1] # output differences if self_value != other_value: @@ -101,90 +118,62 @@ class backtest_lookahead_bias_checker: self.current_analysis.false_indicators.append(col_name[0]) print(f"=> found look ahead bias in indicator {col_name[0]}. " + f"{str(self_value)} != {str(other_value)}") - # return compare_df - - def report_signal(self, result, column_name, checked_timestamp): - df = result['results'] - row_count = df[column_name].shape[0] - if row_count == 0: - return False + def prepare_data(self, varHolder, pairs_to_load): + prepare_data_config = copy.deepcopy(self.local_config) + prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varHolder.from_dt)) + "-" + + str(self.dt_to_timestamp(varHolder.to_dt))) + prepare_data_config['pairs'] = pairs_to_load + self.backtesting = Backtesting(prepare_data_config) + self.backtesting._set_strategy(self.backtesting.strategylist[0]) + varHolder.data, varHolder.timerange = self.backtesting.load_bt_data() + varHolder.indicators = self.backtesting.strategy.advise_all_indicators(varHolder.data) + varHolder.result = self.get_result(self.backtesting, varHolder.indicators) + + def update_output_file(self): + pass + + def start(self, config, strategy_obj: dict, args) -> None: + + # deepcopy so we can change the pairs for the 2ndary runs + # and not worry about another strategy to check after. + self.local_config = deepcopy(config) + self.local_config['strategy_list'] = [strategy_obj['name']] + self.current_analysis = BacktestLookaheadBiasChecker.Analysis() + self.minimum_trade_amount = args['minimum_trade_amount'] + self.targeted_trade_amount = args['targeted_trade_amount'] + + # first make a single backtest + self.full_varHolder = BacktestLookaheadBiasChecker.VarHolder() + + # define datetime in human-readable format + parsed_timerange = TimeRange.parse_timerange(config['timerange']) + if (parsed_timerange is not None and + parsed_timerange.startdt is not None and + parsed_timerange.stopdt is not None): + self.full_varHolder.from_dt = parsed_timerange.startdt + self.full_varHolder.to_dt = parsed_timerange.stopdt else: + print("Parsing of parsed_timerange failed. exiting!") + return - df_cut = df[(df[column_name] == checked_timestamp)] - if df_cut[column_name].shape[0] == 0: - # print("did NOT find the same signal in column " + column_name + - # " at timestamp " + str(checked_timestamp)) - return False - else: - return True - return False + self.prepare_data(self.full_varHolder, self.local_config['pairs']) - def prepare_data(self, varholder, var_pairs): - self.config['timerange'] = \ - str(int(self.dt_to_timestamp(varholder.from_dt))) + "-" + \ - str(int(self.dt_to_timestamp(varholder.to_dt))) - self.backtesting = Backtesting(self.config) - self.backtesting._set_strategy(self.backtesting.strategylist[0]) - varholder.data, varholder.timerange = self.backtesting.load_bt_data() - varholder.indicators = self.backtesting.strategy.advise_all_indicators(varholder.data) - varholder.result = self.get_result(self.backtesting, varholder.indicators) - - def start(self, config, strategy_obj: dict) -> None: - self.strategy_obj = strategy_obj - self.config = config - self.current_analysis = backtest_lookahead_bias_checker.analysis() - - max_try_signals: int = 3 - found_signals: int = 0 - continue_with_strategy = True - - # first we need to get the necessary entry/exit signals - # so we start by 14 days and increase in 1 month steps - # until we have the desired trade amount. - for try_buysignals in range(max_try_signals): # range(3) = 0..2 - # re-initialize backtesting-variable - self.full_varHolder = backtest_lookahead_bias_checker.varHolder() - - # define datetimes in human readable format - self.full_varHolder.from_dt = datetime(2022, 9, 1) - self.full_varHolder.to_dt = datetime(2022, 9, 15) + timedelta(days=30 * try_buysignals) - - self.prepare_data(self.full_varHolder, self.config['pairs']) - - found_signals = self.full_varHolder.result['results'].shape[0] + 1 - if try_buysignals == max_try_signals - 1: - if found_signals < self.signals_to_check / 2: - print(f"... only found {str(int(found_signals / 2))} " - f"buy signals for {self.strategy_obj['name']}. " - f"Cancelling...") - continue_with_strategy = False - else: - print( - f"Found {str(found_signals)} buy signals. " - f"Going with max {str(self.signals_to_check)} " - f" buy signals in the full timerange from " - f"{str(self.full_varHolder.from_dt)} to {str(self.full_varHolder.to_dt)}") - break - elif found_signals < self.signals_to_check: - print( - f"Only found {str(found_signals)} buy signals in the full timerange from " - f"{str(self.full_varHolder.from_dt)} to " - f"{str(self.full_varHolder.to_dt)}. " - f"will increase timerange trying to get at least " - f"{str(self.signals_to_check)} signals.") - else: - print( - f"Found {str(found_signals)} buy signals, more than necessary. " - f"Reducing to {str(self.signals_to_check)} " - f"checked buy signals in the full timerange from " - f"{str(self.full_varHolder.from_dt)} to {str(self.full_varHolder.to_dt)}") - break - if not continue_with_strategy: + found_signals: int = self.full_varHolder.result['results'].shape[0] + 1 + if found_signals >= self.targeted_trade_amount: + print(f"Found {found_signals} trades, calculating {self.targeted_trade_amount} trades.") + elif self.targeted_trade_amount >= found_signals >= self.minimum_trade_amount: + print(f"Only found {found_signals} trades. Calculating all available trades.") + else: + print(f"found {found_signals} trades " + f"which is less than minimum_trade_amount {self.minimum_trade_amount}. " + f"Cancelling this backtest lookahead bias test.") return + # now we loop through all entry signals + # starting from the same datetime to avoid miss-reports of bias for idx, result_row in self.full_varHolder.result['results'].iterrows(): - if self.current_analysis.total_signals == self.signals_to_check: + if self.current_analysis.total_signals == self.targeted_trade_amount: break # if force-sold, ignore this signal since here it will unconditionally exit. @@ -193,49 +182,45 @@ class backtest_lookahead_bias_checker: self.current_analysis.total_signals += 1 - self.entry_varholder = backtest_lookahead_bias_checker.varHolder() - self.exit_varholder = backtest_lookahead_bias_checker.varHolder() - - self.entry_varholder.from_dt = self.full_varHolder.from_dt # result_row['open_date'] - self.entry_varholder.compared_dt = result_row['open_date'] + self.entry_varHolder = BacktestLookaheadBiasChecker.VarHolder() + self.exit_varHolder = BacktestLookaheadBiasChecker.VarHolder() + self.entry_varHolder.from_dt = self.full_varHolder.from_dt + self.entry_varHolder.compared_dt = result_row['open_date'] # to_dt needs +1 candle since it won't buy on the last candle - self.entry_varholder.to_dt = result_row['open_date'] + \ - timedelta(minutes=timeframe_to_minutes(self.config['timeframe']) * 2) + self.entry_varHolder.to_dt = (result_row['open_date'] + + timedelta(minutes=timeframe_to_minutes( + self.local_config['timeframe']))) - self.prepare_data(self.entry_varholder, [result_row['pair']]) + self.prepare_data(self.entry_varHolder, [result_row['pair']]) - # --- - # print("analyzing the sell signal") - # to_dt needs +1 candle since it will always sell all trades on the last candle - self.exit_varholder.from_dt = self.full_varHolder.from_dt # result_row['open_date'] - self.exit_varholder.to_dt = \ - result_row['close_date'] + \ - timedelta(minutes=timeframe_to_minutes(self.config['timeframe'])) - self.exit_varholder.compared_dt = result_row['close_date'] + # to_dt needs +1 candle since it will always exit/force-exit trades on the last candle + self.exit_varHolder.from_dt = self.full_varHolder.from_dt + self.exit_varHolder.to_dt = (result_row['close_date'] + + timedelta(minutes=timeframe_to_minutes( + self.local_config['timeframe']))) + self.exit_varHolder.compared_dt = result_row['close_date'] - self.prepare_data(self.exit_varholder, [result_row['pair']]) + self.prepare_data(self.exit_varHolder, [result_row['pair']]) # register if buy signal is broken if not self.report_signal( - self.entry_varholder.result, - "open_date", self.entry_varholder.compared_dt): + self.entry_varHolder.result, "open_date", self.entry_varHolder.compared_dt): self.current_analysis.false_entry_signals += 1 # register if buy or sell signal is broken - if not self.report_signal(self.entry_varholder.result, - "open_date", self.entry_varholder.compared_dt) \ - or not self.report_signal(self.exit_varholder.result, - "close_date", self.exit_varholder.compared_dt): + if not self.report_signal( + self.exit_varHolder.result, "close_date", self.exit_varHolder.compared_dt): self.current_analysis.false_exit_signals += 1 - self.analyze_indicators(self.full_varHolder, self.entry_varholder, result_row['pair']) - self.analyze_indicators(self.full_varHolder, self.exit_varholder, result_row['pair']) + # check if the indicators themselves contain biased data + self.analyze_indicators(self.full_varHolder, self.entry_varHolder, result_row['pair']) + self.analyze_indicators(self.full_varHolder, self.exit_varHolder, result_row['pair']) - if self.current_analysis.false_entry_signals > 0 or \ - self.current_analysis.false_exit_signals > 0 or \ - len(self.current_analysis.false_indicators) > 0: - print(" => " + self.strategy_obj['name'] + ": bias detected!") + if (self.current_analysis.false_entry_signals > 0 or + self.current_analysis.false_exit_signals > 0 or + len(self.current_analysis.false_indicators) > 0): + print(" => " + self.local_config['strategy_list'][0] + ": bias detected!") self.current_analysis.has_bias = True else: - print(self.strategy_obj['name'] + ": no bias detected") + print(self.local_config['strategy_list'][0] + ": no bias detected") From a9ef4c3ab013b6e7a6f953788ce221b8f5301ea7 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Wed, 12 Apr 2023 21:03:59 +0200 Subject: [PATCH 03/55] partial progress commit: added terminal tabulate-output added yet non-working csv output using pandas --- freqtrade/commands/arguments.py | 10 ++- freqtrade/commands/cli_options.py | 5 ++ freqtrade/commands/strategy_utils_commands.py | 82 ++++++++++++++++--- .../backtest_lookahead_bias_checker.py | 68 ++++++++------- 4 files changed, 118 insertions(+), 47 deletions(-) mode change 100644 => 100755 freqtrade/commands/arguments.py mode change 100644 => 100755 freqtrade/commands/cli_options.py mode change 100644 => 100755 freqtrade/commands/strategy_utils_commands.py mode change 100644 => 100755 freqtrade/strategy/backtest_lookahead_bias_checker.py diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py old mode 100644 new mode 100755 index 6cb727eaf..ac5c33ad1 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -116,10 +116,11 @@ NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] -ARGS_STRATEGY_UPDATER = ["strategy_list"] +ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"] ARGS_BACKTEST_LOOKAHEAD_BIAS_CHECKER = ARGS_BACKTEST + ["minimum_trade_amount", - "targeted_trade_amount"] + "targeted_trade_amount", + "overwrite_existing_exportfilename_content"] # + ["target_trades", "minimum_trades", @@ -458,13 +459,14 @@ class Arguments: 'files to the current version', parents=[_common_parser]) strategy_updater_cmd.set_defaults(func=start_strategy_update) - self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd) + self._build_args(optionlist=ARGS_STRATEGY_UPDATER, + parser=strategy_updater_cmd) # Add backtest lookahead bias checker subcommand backtest_lookahead_bias_checker_cmd = \ subparsers.add_parser('backtest-lookahead-bias-checker', help="checks for potential look ahead bias", - parents=[_common_parser]) + parents=[_common_parser, _strategy_parser]) backtest_lookahead_bias_checker_cmd.set_defaults(func=start_backtest_lookahead_bias_checker) self._build_args(optionlist=ARGS_BACKTEST_LOOKAHEAD_BIAS_CHECKER, diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py old mode 100644 new mode 100755 index 5d2af934f..e0709fc31 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -688,5 +688,10 @@ AVAILABLE_CLI_OPTIONS = { type=check_int_positive, metavar='INT', default=20, + ), + "overwrite_existing_exportfilename_content": Arg( + '--overwrite-existing-exportfilename-content', + help='overwrites existing contents if existent with exportfilename given', + action='store_true' ) } diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py old mode 100644 new mode 100755 index 663ea571a..b46481734 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -4,6 +4,9 @@ import time from pathlib import Path from typing import Any, Dict +import pandas as pd +from tabulate import tabulate + from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.resolvers import StrategyResolver @@ -76,7 +79,6 @@ def start_backtest_lookahead_bias_checker(args: Dict[str, Any]) -> None: config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) bias_checker_instances = [] - filtered_strategy_objs = [] if 'strategy_list' in args and args['strategy_list'] is not None: for args_strategy in args['strategy_list']: @@ -87,28 +89,82 @@ def start_backtest_lookahead_bias_checker(args: Dict[str, Any]) -> None: break for filtered_strategy_obj in filtered_strategy_objs: - bias_checker_instances = initialize_single_lookahead_bias_checker( - filtered_strategy_obj, config, args) + bias_checker_instances.append( + initialize_single_lookahead_bias_checker(filtered_strategy_obj, config, args)) else: processed_locations = set() for strategy_obj in strategy_objs: if strategy_obj['location'] not in processed_locations: processed_locations.add(strategy_obj['location']) - bias_checker_instances = initialize_single_lookahead_bias_checker( - strategy_obj, config, args) - create_result_list(bias_checker_instances) - - -def create_result_list(bias_checker_instances): - pass + bias_checker_instances.append( + initialize_single_lookahead_bias_checker(strategy_obj, config, args)) + text_table_bias_checker_instances(bias_checker_instances) + export_to_csv(args, bias_checker_instances) + + +def text_table_bias_checker_instances(bias_checker_instances): + headers = ['strategy', 'has_bias', + 'total_signals', 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators'] + data = [] + for current_instance in bias_checker_instances: + data.append( + [current_instance.strategy_obj['name'], + current_instance.current_analysis.has_bias, + current_instance.current_analysis.total_signals, + current_instance.current_analysis.false_entry_signals, + current_instance.current_analysis.false_exit_signals, + ", ".join(current_instance.current_analysis.false_indicators)] + ) + table = tabulate(data, headers=headers, tablefmt="orgtbl") + print(table) + + +def export_to_csv(args, bias_checker_instances): + def add_or_update_row(df, row_data): + strategy_col_name = 'strategy' + if row_data[strategy_col_name] in df[strategy_col_name].values: + # create temporary dataframe with a single row + # and use that to replace the previous data in there. + index = (df.index[df[strategy_col_name] == + row_data[strategy_col_name]][0]) + df.loc[index] = pd.Series(row_data, index='strategy') + + else: + df = df.concat(row_data, ignore_index=True) + return df + + csv_df = None + + if not Path.exists(args['exportfilename']): + # If the file doesn't exist, create a new DataFrame from scratch + csv_df = pd.DataFrame(columns=['filename', 'strategy', 'has_bias', + 'total_signals', + 'biased_entry_signals', 'biased_exit_signals', + 'biased_indicators'], + index='filename') + else: + # Read CSV file into a pandas dataframe + csv_df = pd.read_csv(args['exportfilename']) + + for inst in bias_checker_instances: + new_row_data = {'filename': inst.strategy_obj['location'].parts[-1], + 'strategy': inst.strategy_obj['name'], + 'has_bias': inst.current_analysis.has_bias, + 'total_signals': inst.current_analysis.total_signals, + 'biased_entry_signals': inst.current_analysis.false_entry_signals, + 'biased_exit_signals': inst.current_analysis.false_exit_signals, + 'biased_indicators': ", ".join(inst.current_analysis.false_indicators)} + csv_df = add_or_update_row(csv_df, new_row_data) + if len(bias_checker_instances) > 0: + print(f"saving {args['exportfilename']}") + csv_df.to_csv(args['exportfilename']) def initialize_single_lookahead_bias_checker(strategy_obj, config, args): - # try: print(f"Bias test of {Path(strategy_obj['location']).name} started.") - instance_backtest_lookahead_bias_checker = BacktestLookaheadBiasChecker() start = time.perf_counter() - current_instance = instance_backtest_lookahead_bias_checker.start(config, strategy_obj, args) + current_instance = BacktestLookaheadBiasChecker() + current_instance.start(config, strategy_obj, args) elapsed = time.perf_counter() - start print(f"checking look ahead bias via backtests of {Path(strategy_obj['location']).name} " f"took {elapsed:.1f} seconds.") diff --git a/freqtrade/strategy/backtest_lookahead_bias_checker.py b/freqtrade/strategy/backtest_lookahead_bias_checker.py old mode 100644 new mode 100755 index c4a321a4a..c48c3a826 --- a/freqtrade/strategy/backtest_lookahead_bias_checker.py +++ b/freqtrade/strategy/backtest_lookahead_bias_checker.py @@ -2,7 +2,7 @@ import copy from copy import deepcopy from datetime import datetime, timedelta, timezone -import pandas +from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.data.history import get_timerange @@ -10,33 +10,37 @@ from freqtrade.exchange import timeframe_to_minutes from freqtrade.optimize.backtesting import Backtesting +class VarHolder: + timerange: TimeRange + data: DataFrame + indicators: DataFrame + result: DataFrame + compared: DataFrame + from_dt: datetime + to_dt: datetime + compared_dt: datetime + + +class Analysis: + def __init__(self): + self.total_signals = 0 + self.false_entry_signals = 0 + self.false_exit_signals = 0 + self.false_indicators = [] + self.has_bias = False + + total_signals: int + false_entry_signals: int + false_exit_signals: int + + false_indicators: list + has_bias: bool + + class BacktestLookaheadBiasChecker: - class VarHolder: - timerange: TimeRange - data: pandas.DataFrame - indicators: pandas.DataFrame - result: pandas.DataFrame - compared: pandas.DataFrame - from_dt: datetime - to_dt: datetime - compared_dt: datetime - - class Analysis: - def __init__(self): - self.total_signals = 0 - self.false_entry_signals = 0 - self.false_exit_signals = 0 - self.false_indicators = [] - self.has_bias = False - - total_signals: int - false_entry_signals: int - false_exit_signals: int - - false_indicators: list - has_bias: bool def __init__(self): + self.exportfilename = None self.strategy_obj = None self.current_analysis = None self.local_config = None @@ -44,7 +48,6 @@ class BacktestLookaheadBiasChecker: self.entry_varHolder = None self.exit_varHolder = None self.backtesting = None - self.current_analysis = None self.minimum_trade_amount = None self.targeted_trade_amount = None @@ -124,9 +127,12 @@ class BacktestLookaheadBiasChecker: prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varHolder.from_dt)) + "-" + str(self.dt_to_timestamp(varHolder.to_dt))) prepare_data_config['pairs'] = pairs_to_load + self.backtesting = Backtesting(prepare_data_config) self.backtesting._set_strategy(self.backtesting.strategylist[0]) varHolder.data, varHolder.timerange = self.backtesting.load_bt_data() + self.backtesting.load_bt_data_detail() + varHolder.indicators = self.backtesting.strategy.advise_all_indicators(varHolder.data) varHolder.result = self.get_result(self.backtesting, varHolder.indicators) @@ -139,12 +145,14 @@ class BacktestLookaheadBiasChecker: # and not worry about another strategy to check after. self.local_config = deepcopy(config) self.local_config['strategy_list'] = [strategy_obj['name']] - self.current_analysis = BacktestLookaheadBiasChecker.Analysis() + self.current_analysis = Analysis() self.minimum_trade_amount = args['minimum_trade_amount'] self.targeted_trade_amount = args['targeted_trade_amount'] + self.exportfilename = args['exportfilename'] + self.strategy_obj = strategy_obj # first make a single backtest - self.full_varHolder = BacktestLookaheadBiasChecker.VarHolder() + self.full_varHolder = VarHolder() # define datetime in human-readable format parsed_timerange = TimeRange.parse_timerange(config['timerange']) @@ -182,8 +190,8 @@ class BacktestLookaheadBiasChecker: self.current_analysis.total_signals += 1 - self.entry_varHolder = BacktestLookaheadBiasChecker.VarHolder() - self.exit_varHolder = BacktestLookaheadBiasChecker.VarHolder() + self.entry_varHolder = VarHolder() + self.exit_varHolder = VarHolder() self.entry_varHolder.from_dt = self.full_varHolder.from_dt self.entry_varHolder.compared_dt = result_row['open_date'] From 767442198ef2c64d27cb73291aeda4669817cb48 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sat, 15 Apr 2023 14:29:52 +0200 Subject: [PATCH 04/55] saving and updating the csv file now works open ended timeranges now work if a file fails then it will not report as non-bias, but report in the table as error and the csv file will not have it listed. --- freqtrade/commands/strategy_utils_commands.py | 78 +++++++++++-------- .../backtest_lookahead_bias_checker.py | 17 ++-- 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index b46481734..417d98ad5 100755 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -103,48 +103,62 @@ def start_backtest_lookahead_bias_checker(args: Dict[str, Any]) -> None: def text_table_bias_checker_instances(bias_checker_instances): - headers = ['strategy', 'has_bias', + headers = ['filename', 'strategy', 'has_bias', 'total_signals', 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators'] data = [] for current_instance in bias_checker_instances: - data.append( - [current_instance.strategy_obj['name'], - current_instance.current_analysis.has_bias, - current_instance.current_analysis.total_signals, - current_instance.current_analysis.false_entry_signals, - current_instance.current_analysis.false_exit_signals, - ", ".join(current_instance.current_analysis.false_indicators)] - ) + if current_instance.failed_bias_check: + data.append( + [ + current_instance.strategy_obj['location'].parts[-1], + current_instance.strategy_obj['name'], + 'error while checking' + ] + ) + else: + data.append( + [ + current_instance.strategy_obj['location'].parts[-1], + current_instance.strategy_obj['name'], + current_instance.current_analysis.has_bias, + current_instance.current_analysis.total_signals, + current_instance.current_analysis.false_entry_signals, + current_instance.current_analysis.false_exit_signals, + ", ".join(current_instance.current_analysis.false_indicators) + ] + ) table = tabulate(data, headers=headers, tablefmt="orgtbl") print(table) def export_to_csv(args, bias_checker_instances): def add_or_update_row(df, row_data): - strategy_col_name = 'strategy' - if row_data[strategy_col_name] in df[strategy_col_name].values: - # create temporary dataframe with a single row - # and use that to replace the previous data in there. - index = (df.index[df[strategy_col_name] == - row_data[strategy_col_name]][0]) - df.loc[index] = pd.Series(row_data, index='strategy') - + if ( + (df['filename'] == row_data['filename']) & + (df['strategy'] == row_data['strategy']) + ).any(): + # Update existing row + pd_series = pd.DataFrame([row_data]) + df.loc[ + (df['filename'] == row_data['filename']) & + (df['strategy'] == row_data['strategy']) + ] = pd_series else: - df = df.concat(row_data, ignore_index=True) - return df + # Add new row + df = pd.concat([df, pd.DataFrame([row_data], columns=df.columns)]) - csv_df = None + return df - if not Path.exists(args['exportfilename']): - # If the file doesn't exist, create a new DataFrame from scratch - csv_df = pd.DataFrame(columns=['filename', 'strategy', 'has_bias', - 'total_signals', - 'biased_entry_signals', 'biased_exit_signals', - 'biased_indicators'], - index='filename') - else: + if Path(args['exportfilename']).exists(): # Read CSV file into a pandas dataframe csv_df = pd.read_csv(args['exportfilename']) + else: + # Create a new empty DataFrame with the desired column names and set the index + csv_df = pd.DataFrame(columns=[ + 'filename', 'strategy', 'has_bias', 'total_signals', + 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators' + ], + index=None) for inst in bias_checker_instances: new_row_data = {'filename': inst.strategy_obj['location'].parts[-1], @@ -153,11 +167,11 @@ def export_to_csv(args, bias_checker_instances): 'total_signals': inst.current_analysis.total_signals, 'biased_entry_signals': inst.current_analysis.false_entry_signals, 'biased_exit_signals': inst.current_analysis.false_exit_signals, - 'biased_indicators': ", ".join(inst.current_analysis.false_indicators)} + 'biased_indicators': ",".join(inst.current_analysis.false_indicators)} csv_df = add_or_update_row(csv_df, new_row_data) - if len(bias_checker_instances) > 0: - print(f"saving {args['exportfilename']}") - csv_df.to_csv(args['exportfilename']) + + print(f"saving {args['exportfilename']}") + csv_df.to_csv(args['exportfilename'], index=False) def initialize_single_lookahead_bias_checker(strategy_obj, config, args): diff --git a/freqtrade/strategy/backtest_lookahead_bias_checker.py b/freqtrade/strategy/backtest_lookahead_bias_checker.py index c48c3a826..98b82e209 100755 --- a/freqtrade/strategy/backtest_lookahead_bias_checker.py +++ b/freqtrade/strategy/backtest_lookahead_bias_checker.py @@ -50,6 +50,7 @@ class BacktestLookaheadBiasChecker: self.backtesting = None self.minimum_trade_amount = None self.targeted_trade_amount = None + self.failed_bias_check = True @staticmethod def dt_to_timestamp(dt): @@ -156,14 +157,16 @@ class BacktestLookaheadBiasChecker: # define datetime in human-readable format parsed_timerange = TimeRange.parse_timerange(config['timerange']) - if (parsed_timerange is not None and - parsed_timerange.startdt is not None and - parsed_timerange.stopdt is not None): + + if parsed_timerange.startdt is None: + self.full_varHolder.from_dt = datetime.utcfromtimestamp(0) + else: self.full_varHolder.from_dt = parsed_timerange.startdt - self.full_varHolder.to_dt = parsed_timerange.stopdt + + if parsed_timerange.stopdt is None: + self.full_varHolder.to_dt = datetime.now() else: - print("Parsing of parsed_timerange failed. exiting!") - return + self.full_varHolder.to_dt = parsed_timerange.stopdt self.prepare_data(self.full_varHolder, self.local_config['pairs']) @@ -232,3 +235,5 @@ class BacktestLookaheadBiasChecker: self.current_analysis.has_bias = True else: print(self.local_config['strategy_list'][0] + ": no bias detected") + + self.failed_bias_check = False From 2b416d3b62159a4e789d5bf5124d3641b651405a Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 16 Apr 2023 23:47:10 +0200 Subject: [PATCH 05/55] - Added a first version of docs (needs checking) - optimized pairs for entry_varholder and exit_varholder to only check a single pair instead of all pairs. - bias-check of freqai strategies now possible - added condition to not crash when compared_df is empty (meaning no differences have been found) --- docs/utils.md | 33 ++++++++++++ freqtrade/commands/strategy_utils_commands.py | 6 +++ .../backtest_lookahead_bias_checker.py | 51 ++++++++++++------- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index eb675442f..cb77ca449 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -999,3 +999,36 @@ Common arguments: Path to userdata directory. ``` +### Backtest lookahead bias checker +#### Summary +Checks a given strategy for look ahead bias +Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting +and producing false hopes for the one backtesting. + +#### Introduction: +Many strategies - without the programmer knowing - have fallen prey to look ahead bias. + +Any backtest will populate the full dataframe including all time stamps at the beginning. +If the programmer is not careful or oblivious how things work internally +(which sometimes can be really hard to find out) then it will just look into the future making the strategy amazing +but not realistic. + +The tool is made to try to verify the validity in the form of the aforementioned look ahead bias. + +#### How does the command work? +It will not look at the strategy or any contents itself but instead will run multiple backtests +by using precisely cut timeranges and analyzing the results each time, comparing to the full timerange. + +At first, it starts a backtest over the whole duration +and then repeats backtests from the same starting point to the respective points to watch. +In addition, it analyzes the dataframes form the overall backtest to the cut ones. + +At the end it will return a result-table in terminal. + +Hint: +If an entry or exit condition is only triggered rarely or the timerange was chosen +so only a few entry conditions are met +then the bias checker is unable to catch the biased entry or exit condition. +In the end it only checks which entry and exit signals have been triggered. + +---Flow chart here for better understanding--- diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 417d98ad5..ab31cfa82 100755 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -91,6 +91,12 @@ def start_backtest_lookahead_bias_checker(args: Dict[str, Any]) -> None: for filtered_strategy_obj in filtered_strategy_objs: bias_checker_instances.append( initialize_single_lookahead_bias_checker(filtered_strategy_obj, config, args)) + elif 'strategy' in args and args['strategy'] is not None: + for strategy_obj in strategy_objs: + if strategy_obj['name'] == args['strategy']: + bias_checker_instances.append( + initialize_single_lookahead_bias_checker(strategy_obj, config, args)) + break else: processed_locations = set() for strategy_obj in strategy_objs: diff --git a/freqtrade/strategy/backtest_lookahead_bias_checker.py b/freqtrade/strategy/backtest_lookahead_bias_checker.py index 98b82e209..2e5ef4165 100755 --- a/freqtrade/strategy/backtest_lookahead_bias_checker.py +++ b/freqtrade/strategy/backtest_lookahead_bias_checker.py @@ -1,4 +1,6 @@ import copy +import pathlib +import shutil from copy import deepcopy from datetime import datetime, timedelta, timezone @@ -45,8 +47,11 @@ class BacktestLookaheadBiasChecker: self.current_analysis = None self.local_config = None self.full_varHolder = None + self.entry_varHolder = None self.exit_varHolder = None + self.entry_varHolders = [] + self.exit_varHolders = [] self.backtesting = None self.minimum_trade_amount = None self.targeted_trade_amount = None @@ -105,29 +110,36 @@ class BacktestLookaheadBiasChecker: if cut_df_cut.shape[0] != 0: compare_df = full_df_cut.compare(cut_df_cut) - # skippedColumns = ["date", "open", "high", "low", "close", "volume"] - for col_name, values in compare_df.items(): - col_idx = compare_df.columns.get_loc(col_name) - compare_df_row = compare_df.iloc[0] - # compare_df now comprises tuples with [1] having either 'self' or 'other' - if 'other' in col_name[1]: - continue - self_value = compare_df_row[col_idx] - other_value = compare_df_row[col_idx + 1] + if compare_df.shape[0] > 0: + for col_name, values in compare_df.items(): + col_idx = compare_df.columns.get_loc(col_name) + compare_df_row = compare_df.iloc[0] + # compare_df now comprises tuples with [1] having either 'self' or 'other' + if 'other' in col_name[1]: + continue + self_value = compare_df_row[col_idx] + other_value = compare_df_row[col_idx + 1] - # output differences - if self_value != other_value: + # output differences + if self_value != other_value: - if not self.current_analysis.false_indicators.__contains__(col_name[0]): - self.current_analysis.false_indicators.append(col_name[0]) - print(f"=> found look ahead bias in indicator {col_name[0]}. " + - f"{str(self_value)} != {str(other_value)}") + if not self.current_analysis.false_indicators.__contains__(col_name[0]): + self.current_analysis.false_indicators.append(col_name[0]) + print(f"=> found look ahead bias in indicator {col_name[0]}. " + + f"{str(self_value)} != {str(other_value)}") def prepare_data(self, varHolder, pairs_to_load): + + # purge previous data + abs_folder_path = pathlib.Path("user_data/models/uniqe-id").resolve() + # remove folder and its contents + if pathlib.Path.exists(abs_folder_path): + shutil.rmtree(abs_folder_path) + prepare_data_config = copy.deepcopy(self.local_config) prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varHolder.from_dt)) + "-" + str(self.dt_to_timestamp(varHolder.to_dt))) - prepare_data_config['pairs'] = pairs_to_load + prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load self.backtesting = Backtesting(prepare_data_config) self.backtesting._set_strategy(self.backtesting.strategylist[0]) @@ -137,9 +149,6 @@ class BacktestLookaheadBiasChecker: varHolder.indicators = self.backtesting.strategy.advise_all_indicators(varHolder.data) varHolder.result = self.get_result(self.backtesting, varHolder.indicators) - def update_output_file(self): - pass - def start(self, config, strategy_obj: dict, args) -> None: # deepcopy so we can change the pairs for the 2ndary runs @@ -195,6 +204,8 @@ class BacktestLookaheadBiasChecker: self.entry_varHolder = VarHolder() self.exit_varHolder = VarHolder() + self.entry_varHolders.append(self.entry_varHolder) + self.exit_varHolders.append(self.exit_varHolder) self.entry_varHolder.from_dt = self.full_varHolder.from_dt self.entry_varHolder.compared_dt = result_row['open_date'] @@ -224,6 +235,8 @@ class BacktestLookaheadBiasChecker: self.exit_varHolder.result, "close_date", self.exit_varHolder.compared_dt): self.current_analysis.false_exit_signals += 1 + if len(self.entry_varHolders) >= 10: + pass # check if the indicators themselves contain biased data self.analyze_indicators(self.full_varHolder, self.entry_varHolder, result_row['pair']) self.analyze_indicators(self.full_varHolder, self.exit_varHolder, result_row['pair']) From 2306c74dc19c6c726c0e5137da13d9ecf9ffbd82 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sat, 6 May 2023 21:56:11 +0200 Subject: [PATCH 06/55] adjusted code to matthias' specifications did not change the code so that it only loads data once yet. --- docs/utils.md | 4 +- freqtrade/commands/__init__.py | 6 +- freqtrade/commands/arguments.py | 26 +- freqtrade/commands/optimize_commands.py | 50 +++ freqtrade/commands/strategy_utils_commands.py | 136 ------- freqtrade/optimize/lookahead_analysis.py | 347 ++++++++++++++++++ 6 files changed, 415 insertions(+), 154 deletions(-) create mode 100755 freqtrade/optimize/lookahead_analysis.py diff --git a/docs/utils.md b/docs/utils.md index cb77ca449..cf8d23865 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -999,9 +999,9 @@ Common arguments: Path to userdata directory. ``` -### Backtest lookahead bias checker +### Lookahead - analysis #### Summary -Checks a given strategy for look ahead bias +Checks a given strategy for look ahead bias via backtest-analysis Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting and producing false hopes for the one backtesting. diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 8add45241..b9346fd5f 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -19,10 +19,10 @@ from freqtrade.commands.list_commands import (start_list_exchanges, start_list_f start_list_markets, start_list_strategies, start_list_timeframes, start_show_trades) from freqtrade.commands.optimize_commands import (start_backtesting, start_backtesting_show, - start_edge, start_hyperopt) + start_edge, start_hyperopt, + start_lookahead_analysis) from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit -from freqtrade.commands.strategy_utils_commands import (start_backtest_lookahead_bias_checker, - start_strategy_update) +from freqtrade.commands.strategy_utils_commands import start_strategy_update from freqtrade.commands.trade_commands import start_trading from freqtrade.commands.webserver_commands import start_webserver diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index ac5c33ad1..59ba0bedb 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -118,9 +118,9 @@ NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"] -ARGS_BACKTEST_LOOKAHEAD_BIAS_CHECKER = ARGS_BACKTEST + ["minimum_trade_amount", - "targeted_trade_amount", - "overwrite_existing_exportfilename_content"] +ARGS_LOOKAHEAD_ANALYSIS = ARGS_BACKTEST + ["minimum_trade_amount", + "targeted_trade_amount", + "overwrite_existing_exportfilename_content"] # + ["target_trades", "minimum_trades", @@ -200,8 +200,7 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.commands import (start_analysis_entries_exits, - start_backtest_lookahead_bias_checker, start_backtesting, + from freqtrade.commands import (start_analysis_entries_exits, start_backtesting, start_backtesting_show, start_convert_data, start_convert_db, start_convert_trades, start_create_userdir, start_download_data, start_edge, @@ -209,8 +208,9 @@ class Arguments: start_install_ui, start_list_data, start_list_exchanges, start_list_freqAI_models, start_list_markets, start_list_strategies, start_list_timeframes, - start_new_config, start_new_strategy, start_plot_dataframe, - start_plot_profit, start_show_trades, start_strategy_update, + start_lookahead_analysis, start_new_config, + start_new_strategy, start_plot_dataframe, start_plot_profit, + start_show_trades, start_strategy_update, start_test_pairlist, start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', @@ -462,12 +462,12 @@ class Arguments: self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd) - # Add backtest lookahead bias checker subcommand - backtest_lookahead_bias_checker_cmd = \ - subparsers.add_parser('backtest-lookahead-bias-checker', + # Add lookahead_analysis subcommand + lookahead_analayis_cmd = \ + subparsers.add_parser('lookahead-analysis', help="checks for potential look ahead bias", parents=[_common_parser, _strategy_parser]) - backtest_lookahead_bias_checker_cmd.set_defaults(func=start_backtest_lookahead_bias_checker) + lookahead_analayis_cmd.set_defaults(func=start_lookahead_analysis) - self._build_args(optionlist=ARGS_BACKTEST_LOOKAHEAD_BIAS_CHECKER, - parser=backtest_lookahead_bias_checker_cmd) + self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS, + parser=lookahead_analayis_cmd) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 1bfd384fc..765f2caf2 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -6,6 +6,8 @@ from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import round_coin_value +from freqtrade.optimize.lookahead_analysis import LookaheadAnalysisSubFunctions +from freqtrade.resolvers import StrategyResolver logger = logging.getLogger(__name__) @@ -132,3 +134,51 @@ def start_edge(args: Dict[str, Any]) -> None: # Initialize Edge object edge_cli = EdgeCli(config) edge_cli.start() + + +def start_lookahead_analysis(args: Dict[str, Any]) -> None: + """ + Start the backtest bias tester script + :param args: Cli args from Arguments() + :return: None + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if args['targeted_trade_amount'] < args['minimum_trade_amount']: + # add logic that tells the user to check the configuration + # since this combo doesn't make any sense. + pass + + strategy_objs = StrategyResolver.search_all_objects( + config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) + + lookaheadAnalysis_instances = [] + strategy_list = [] + + # unify --strategy and --strategy_list to one list + if 'strategy' in args and args['strategy'] is not None: + strategy_list = [args['strategy']] + else: + strategy_list = args['strategy_list'] + + # check if strategies can be properly loaded, only check them if they can be. + if strategy_list is not None: + for strat in strategy_list: + for strategy_obj in strategy_objs: + if strategy_obj['name'] == strat and strategy_obj not in strategy_list: + lookaheadAnalysis_instances.append( + LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( + strategy_obj, config, args)) + break + + # report the results + if lookaheadAnalysis_instances: + LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + lookaheadAnalysis_instances) + if args['exportfilename'] is not None: + LookaheadAnalysisSubFunctions.export_to_csv(args, lookaheadAnalysis_instances) + else: + logger.error("There were no strategies specified neither through " + "--strategy nor through " + "--strategy_list " + "or timeframe was not specified.") diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index ab31cfa82..e579ec475 100755 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -4,13 +4,9 @@ import time from pathlib import Path from typing import Any, Dict -import pandas as pd -from tabulate import tabulate - from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.resolvers import StrategyResolver -from freqtrade.strategy.backtest_lookahead_bias_checker import BacktestLookaheadBiasChecker from freqtrade.strategy.strategyupdater import StrategyUpdater @@ -57,135 +53,3 @@ def start_conversion(strategy_obj, config): instance_strategy_updater.start(config, strategy_obj) elapsed = time.perf_counter() - start print(f"Conversion of {Path(strategy_obj['location']).name} took {elapsed:.1f} seconds.") - - # except: - # pass - - -def start_backtest_lookahead_bias_checker(args: Dict[str, Any]) -> None: - """ - Start the backtest bias tester script - :param args: Cli args from Arguments() - :return: None - """ - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if args['targeted_trade_amount'] < args['minimum_trade_amount']: - # add logic that tells the user to check the configuration - # since this combo doesn't make any sense. - pass - - strategy_objs = StrategyResolver.search_all_objects( - config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) - - bias_checker_instances = [] - filtered_strategy_objs = [] - if 'strategy_list' in args and args['strategy_list'] is not None: - for args_strategy in args['strategy_list']: - for strategy_obj in strategy_objs: - if (strategy_obj['name'] == args_strategy - and strategy_obj not in filtered_strategy_objs): - filtered_strategy_objs.append(strategy_obj) - break - - for filtered_strategy_obj in filtered_strategy_objs: - bias_checker_instances.append( - initialize_single_lookahead_bias_checker(filtered_strategy_obj, config, args)) - elif 'strategy' in args and args['strategy'] is not None: - for strategy_obj in strategy_objs: - if strategy_obj['name'] == args['strategy']: - bias_checker_instances.append( - initialize_single_lookahead_bias_checker(strategy_obj, config, args)) - break - else: - processed_locations = set() - for strategy_obj in strategy_objs: - if strategy_obj['location'] not in processed_locations: - processed_locations.add(strategy_obj['location']) - bias_checker_instances.append( - initialize_single_lookahead_bias_checker(strategy_obj, config, args)) - text_table_bias_checker_instances(bias_checker_instances) - export_to_csv(args, bias_checker_instances) - - -def text_table_bias_checker_instances(bias_checker_instances): - headers = ['filename', 'strategy', 'has_bias', - 'total_signals', 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators'] - data = [] - for current_instance in bias_checker_instances: - if current_instance.failed_bias_check: - data.append( - [ - current_instance.strategy_obj['location'].parts[-1], - current_instance.strategy_obj['name'], - 'error while checking' - ] - ) - else: - data.append( - [ - current_instance.strategy_obj['location'].parts[-1], - current_instance.strategy_obj['name'], - current_instance.current_analysis.has_bias, - current_instance.current_analysis.total_signals, - current_instance.current_analysis.false_entry_signals, - current_instance.current_analysis.false_exit_signals, - ", ".join(current_instance.current_analysis.false_indicators) - ] - ) - table = tabulate(data, headers=headers, tablefmt="orgtbl") - print(table) - - -def export_to_csv(args, bias_checker_instances): - def add_or_update_row(df, row_data): - if ( - (df['filename'] == row_data['filename']) & - (df['strategy'] == row_data['strategy']) - ).any(): - # Update existing row - pd_series = pd.DataFrame([row_data]) - df.loc[ - (df['filename'] == row_data['filename']) & - (df['strategy'] == row_data['strategy']) - ] = pd_series - else: - # Add new row - df = pd.concat([df, pd.DataFrame([row_data], columns=df.columns)]) - - return df - - if Path(args['exportfilename']).exists(): - # Read CSV file into a pandas dataframe - csv_df = pd.read_csv(args['exportfilename']) - else: - # Create a new empty DataFrame with the desired column names and set the index - csv_df = pd.DataFrame(columns=[ - 'filename', 'strategy', 'has_bias', 'total_signals', - 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators' - ], - index=None) - - for inst in bias_checker_instances: - new_row_data = {'filename': inst.strategy_obj['location'].parts[-1], - 'strategy': inst.strategy_obj['name'], - 'has_bias': inst.current_analysis.has_bias, - 'total_signals': inst.current_analysis.total_signals, - 'biased_entry_signals': inst.current_analysis.false_entry_signals, - 'biased_exit_signals': inst.current_analysis.false_exit_signals, - 'biased_indicators': ",".join(inst.current_analysis.false_indicators)} - csv_df = add_or_update_row(csv_df, new_row_data) - - print(f"saving {args['exportfilename']}") - csv_df.to_csv(args['exportfilename'], index=False) - - -def initialize_single_lookahead_bias_checker(strategy_obj, config, args): - print(f"Bias test of {Path(strategy_obj['location']).name} started.") - start = time.perf_counter() - current_instance = BacktestLookaheadBiasChecker() - current_instance.start(config, strategy_obj, args) - elapsed = time.perf_counter() - start - print(f"checking look ahead bias via backtests of {Path(strategy_obj['location']).name} " - f"took {elapsed:.1f} seconds.") - return current_instance diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py new file mode 100755 index 000000000..fa8cd5822 --- /dev/null +++ b/freqtrade/optimize/lookahead_analysis.py @@ -0,0 +1,347 @@ +import copy +import logging +import pathlib +import shutil +import time +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, List + +import pandas as pd + +from freqtrade.configuration import TimeRange +from freqtrade.data.history import get_timerange +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.optimize.backtesting import Backtesting + + +logger = logging.getLogger(__name__) + + +class VarHolder: + timerange: TimeRange + data: pd.DataFrame + indicators: pd.DataFrame + result: pd.DataFrame + compared: pd.DataFrame + from_dt: datetime + to_dt: datetime + compared_dt: datetime + timeframe: str + + +class Analysis: + def __init__(self) -> None: + self.total_signals = 0 + self.false_entry_signals = 0 + self.false_exit_signals = 0 + self.false_indicators: List[str] = [] + self.has_bias = False + + +class LookaheadAnalysis: + + def __init__(self, config: Dict[str, Any], strategy_obj: dict, args: Dict[str, Any]): + self.failed_bias_check = True + self.full_varHolder = VarHolder + + self.entry_varHolders: List[VarHolder] = [] + self.exit_varHolders: List[VarHolder] = [] + + # pull variables the scope of the lookahead_analysis-instance + self.local_config = deepcopy(config) + self.local_config['strategy'] = strategy_obj['name'] + self.current_analysis = Analysis() + self.minimum_trade_amount = args['minimum_trade_amount'] + self.targeted_trade_amount = args['targeted_trade_amount'] + self.exportfilename = args['exportfilename'] + self.strategy_obj = strategy_obj + + @staticmethod + def dt_to_timestamp(dt: datetime): + timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) + return timestamp + + @staticmethod + def get_result(backtesting, processed: pd.DataFrame): + min_date, max_date = get_timerange(processed) + + result = backtesting.backtest( + processed=deepcopy(processed), + start_date=min_date, + end_date=max_date + ) + return result + + @staticmethod + def report_signal(result: dict, column_name: str, checked_timestamp: datetime): + df = result['results'] + row_count = df[column_name].shape[0] + + if row_count == 0: + return False + else: + + df_cut = df[(df[column_name] == checked_timestamp)] + if df_cut[column_name].shape[0] == 0: + return False + else: + return True + return False + + # analyzes two data frames with processed indicators and shows differences between them. + def analyze_indicators(self, full_vars: VarHolder, cut_vars: VarHolder, current_pair): + # extract dataframes + cut_df = cut_vars.indicators[current_pair] + full_df = full_vars.indicators[current_pair] + + # cut longer dataframe to length of the shorter + full_df_cut = full_df[ + (full_df.date == cut_vars.compared_dt) + ].reset_index(drop=True) + cut_df_cut = cut_df[ + (cut_df.date == cut_vars.compared_dt) + ].reset_index(drop=True) + + # compare dataframes + if full_df_cut.shape[0] != 0: + if cut_df_cut.shape[0] != 0: + compare_df = full_df_cut.compare(cut_df_cut) + + if compare_df.shape[0] > 0: + for col_name, values in compare_df.items(): + col_idx = compare_df.columns.get_loc(col_name) + compare_df_row = compare_df.iloc[0] + # compare_df now comprises tuples with [1] having either 'self' or 'other' + if 'other' in col_name[1]: + continue + self_value = compare_df_row[col_idx] + other_value = compare_df_row[col_idx + 1] + + # output differences + if self_value != other_value: + + if not self.current_analysis.false_indicators.__contains__(col_name[0]): + self.current_analysis.false_indicators.append(col_name[0]) + logging.info(f"=> found look ahead bias in indicator " + f"{col_name[0]}. " + f"{str(self_value)} != {str(other_value)}") + + def prepare_data(self, varholder: VarHolder, pairs_to_load: List[pd.DataFrame]): + + # purge previous data + abs_folder_path = pathlib.Path("user_data/models/uniqe-id").resolve() + # remove folder and its contents + if pathlib.Path.exists(abs_folder_path): + shutil.rmtree(abs_folder_path) + + prepare_data_config = copy.deepcopy(self.local_config) + prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + + str(self.dt_to_timestamp(varholder.to_dt))) + prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load + + self.backtesting = Backtesting(prepare_data_config) + self.backtesting._set_strategy(self.backtesting.strategylist[0]) + varholder.data, varholder.timerange = self.backtesting.load_bt_data() + self.backtesting.load_bt_data_detail() + varholder.timeframe = self.backtesting.timeframe + + varholder.indicators = self.backtesting.strategy.advise_all_indicators(varholder.data) + varholder.result = self.get_result(self.backtesting, varholder.indicators) + + def fill_full_varholder(self): + self.full_varHolder = VarHolder() + + # define datetime in human-readable format + parsed_timerange = TimeRange.parse_timerange(self.local_config['timerange']) + + if parsed_timerange.startdt is None: + self.full_varHolder.from_dt = datetime.fromtimestamp(0, tz=timezone.utc) + else: + self.full_varHolder.from_dt = parsed_timerange.startdt + + if parsed_timerange.stopdt is None: + self.full_varHolder.to_dt = datetime.utcnow() + else: + self.full_varHolder.to_dt = parsed_timerange.stopdt + + self.prepare_data(self.full_varHolder, self.local_config['pairs']) + + def fill_entry_and_exit_varHolders(self, idx, result_row): + # entry_varHolder + entry_varHolder = VarHolder() + self.entry_varHolders.append(entry_varHolder) + entry_varHolder.from_dt = self.full_varHolder.from_dt + entry_varHolder.compared_dt = result_row['open_date'] + # to_dt needs +1 candle since it won't buy on the last candle + entry_varHolder.to_dt = ( + result_row['open_date'] + + timedelta(minutes=timeframe_to_minutes(self.full_varHolder.timeframe))) + self.prepare_data(entry_varHolder, [result_row['pair']]) + + # exit_varHolder + exit_varHolder = VarHolder() + self.exit_varHolders.append(exit_varHolder) + # to_dt needs +1 candle since it will always exit/force-exit trades on the last candle + exit_varHolder.from_dt = self.full_varHolder.from_dt + exit_varHolder.to_dt = ( + result_row['close_date'] + + timedelta(minutes=timeframe_to_minutes(self.full_varHolder.timeframe))) + exit_varHolder.compared_dt = result_row['close_date'] + self.prepare_data(exit_varHolder, [result_row['pair']]) + + # now we analyze a full trade of full_varholder and look for analyze its bias + def analyze_row(self, idx, result_row): + # if force-sold, ignore this signal since here it will unconditionally exit. + if result_row.close_date == self.dt_to_timestamp(self.full_varHolder.to_dt): + return + + # keep track of how many signals are processed at total + self.current_analysis.total_signals += 1 + + # fill entry_varHolder and exit_varHolder + self.fill_entry_and_exit_varHolders(idx, result_row) + + # register if buy signal is broken + if not self.report_signal( + self.entry_varHolders[idx].result, + "open_date", + self.entry_varHolders[idx].compared_dt): + self.current_analysis.false_entry_signals += 1 + + # register if buy or sell signal is broken + if not self.report_signal( + self.exit_varHolders[idx].result, + "close_date", + self.exit_varHolders[idx].compared_dt): + self.current_analysis.false_exit_signals += 1 + + # check if the indicators themselves contain biased data + self.analyze_indicators(self.full_varHolder, self.entry_varHolders[idx], result_row['pair']) + self.analyze_indicators(self.full_varHolder, self.exit_varHolders[idx], result_row['pair']) + + def start(self) -> None: + + # first make a single backtest + self.fill_full_varholder() + + # check if requirements have been met of full_varholder + found_signals: int = self.full_varHolder.result['results'].shape[0] + 1 + if found_signals >= self.targeted_trade_amount: + logging.info(f"Found {found_signals} trades, " + f"calculating {self.targeted_trade_amount} trades.") + elif self.targeted_trade_amount >= found_signals >= self.minimum_trade_amount: + logging.info(f"Only found {found_signals} trades. Calculating all available trades.") + else: + logging.info(f"found {found_signals} trades " + f"which is less than minimum_trade_amount {self.minimum_trade_amount}. " + f"Cancelling this backtest lookahead bias test.") + return + + # now we loop through all signals + # starting from the same datetime to avoid miss-reports of bias + for idx, result_row in self.full_varHolder.result['results'].iterrows(): + if self.current_analysis.total_signals == self.targeted_trade_amount: + break + self.analyze_row(idx, result_row) + + # check and report signals + if (self.current_analysis.false_entry_signals > 0 or + self.current_analysis.false_exit_signals > 0 or + len(self.current_analysis.false_indicators) > 0): + logging.info(f" => {self.local_config['strategy']} + : bias detected!") + self.current_analysis.has_bias = True + else: + logging.info(self.local_config['strategy'] + ": no bias detected") + + self.failed_bias_check = False + + +class LookaheadAnalysisSubFunctions: + @staticmethod + def text_table_lookahead_analysis_instances(lookahead_instances: List[LookaheadAnalysis]): + headers = ['filename', 'strategy', 'has_bias', 'total_signals', + 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators'] + data = [] + for inst in lookahead_instances: + if inst.failed_bias_check: + data.append( + [ + inst.strategy_obj['location'].parts[-1], + inst.strategy_obj['name'], + 'error while checking' + ] + ) + else: + data.append( + [ + inst.strategy_obj['location'].parts[-1], + inst.strategy_obj['name'], + inst.current_analysis.has_bias, + inst.current_analysis.total_signals, + inst.current_analysis.false_entry_signals, + inst.current_analysis.false_exit_signals, + ", ".join(inst.current_analysis.false_indicators) + ] + ) + from tabulate import tabulate + table = tabulate(data, headers=headers, tablefmt="orgtbl") + print(table) + + @staticmethod + def export_to_csv(args: Dict[str, Any], lookahead_analysis: List[LookaheadAnalysis]): + def add_or_update_row(df, row_data): + if ( + (df['filename'] == row_data['filename']) & + (df['strategy'] == row_data['strategy']) + ).any(): + # Update existing row + pd_series = pd.DataFrame([row_data]) + df.loc[ + (df['filename'] == row_data['filename']) & + (df['strategy'] == row_data['strategy']) + ] = pd_series + else: + # Add new row + df = pd.concat([df, pd.DataFrame([row_data], columns=df.columns)]) + + return df + + if Path(args['exportfilename']).exists(): + # Read CSV file into a pandas dataframe + csv_df = pd.read_csv(args['exportfilename']) + else: + # Create a new empty DataFrame with the desired column names and set the index + csv_df = pd.DataFrame(columns=[ + 'filename', 'strategy', 'has_bias', 'total_signals', + 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators' + ], + index=None) + + for inst in lookahead_analysis: + new_row_data = {'filename': inst.strategy_obj['location'].parts[-1], + 'strategy': inst.strategy_obj['name'], + 'has_bias': inst.current_analysis.has_bias, + 'total_signals': inst.current_analysis.total_signals, + 'biased_entry_signals': inst.current_analysis.false_entry_signals, + 'biased_exit_signals': inst.current_analysis.false_exit_signals, + 'biased_indicators': ",".join(inst.current_analysis.false_indicators)} + csv_df = add_or_update_row(csv_df, new_row_data) + + logger.info(f"saving {args['exportfilename']}") + csv_df.to_csv(args['exportfilename'], index=False) + + @staticmethod + def initialize_single_lookahead_analysis(strategy_obj: Dict[str, Any], config: Dict[str, Any], + args: Dict[str, Any]): + + logger.info(f"Bias test of {Path(strategy_obj['location']).name} started.") + start = time.perf_counter() + current_instance = LookaheadAnalysis(config, strategy_obj, args) + current_instance.start() + elapsed = time.perf_counter() - start + logger.info(f"checking look ahead bias via backtests " + f"of {Path(strategy_obj['location']).name} " + f"took {elapsed:.0f} seconds.") + return current_instance From b252bdd3c7e15956920ff7b308265cdfb4c45afc Mon Sep 17 00:00:00 2001 From: hippocritical Date: Mon, 8 May 2023 22:35:13 +0200 Subject: [PATCH 07/55] made purging of config.freqai.identifier variable --- freqtrade/optimize/lookahead_analysis.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index fa8cd5822..3ca1b71a0 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -130,11 +130,16 @@ class LookaheadAnalysis: def prepare_data(self, varholder: VarHolder, pairs_to_load: List[pd.DataFrame]): - # purge previous data - abs_folder_path = pathlib.Path("user_data/models/uniqe-id").resolve() - # remove folder and its contents - if pathlib.Path.exists(abs_folder_path): - shutil.rmtree(abs_folder_path) + if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: + # purge previous data if the freqai model is defined + # (to be sure nothing is carried over from older backtests) + path_to_current_identifier = ( + pathlib.Path(f"{self.local_config['user_data_dir']}" + "/models/" + f"{self.local_config['freqai']['identifier']}").resolve()) + # remove folder and its contents + if pathlib.Path.exists(path_to_current_identifier): + shutil.rmtree(path_to_current_identifier) prepare_data_config = copy.deepcopy(self.local_config) prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + @@ -143,6 +148,7 @@ class LookaheadAnalysis: self.backtesting = Backtesting(prepare_data_config) self.backtesting._set_strategy(self.backtesting.strategylist[0]) + varholder.data, varholder.timerange = self.backtesting.load_bt_data() self.backtesting.load_bt_data_detail() varholder.timeframe = self.backtesting.timeframe @@ -168,7 +174,7 @@ class LookaheadAnalysis: self.prepare_data(self.full_varHolder, self.local_config['pairs']) - def fill_entry_and_exit_varHolders(self, idx, result_row): + def fill_entry_and_exit_varHolders(self, result_row): # entry_varHolder entry_varHolder = VarHolder() self.entry_varHolders.append(entry_varHolder) @@ -201,7 +207,7 @@ class LookaheadAnalysis: self.current_analysis.total_signals += 1 # fill entry_varHolder and exit_varHolder - self.fill_entry_and_exit_varHolders(idx, result_row) + self.fill_entry_and_exit_varHolders(result_row) # register if buy signal is broken if not self.report_signal( From 91ce1cb2aeab0ff332bb2a0b3a6823c426c571cf Mon Sep 17 00:00:00 2001 From: hippocritical Date: Wed, 10 May 2023 22:41:27 +0200 Subject: [PATCH 08/55] removed overwrite_existing_exportfilename_content (won't use it myself, wouldn't make sense for others to not overwrite something they re-calculated) switched from args to config (args still work) renamed exportfilename to lookahead_analysis_exportfilename so if users decide to put something into it then it won't compete with other configurations --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 11 ++++++----- freqtrade/commands/optimize_commands.py | 4 ++-- freqtrade/configuration/configuration.py | 6 +++++- freqtrade/optimize/lookahead_analysis.py | 10 +++++----- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index f0419a7d9..af5f0a470 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -121,7 +121,7 @@ ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_s ARGS_LOOKAHEAD_ANALYSIS = ARGS_BACKTEST + ["minimum_trade_amount", "targeted_trade_amount", - "overwrite_existing_exportfilename_content"] + "lookahead_analysis_exportfilename"] # + ["target_trades", "minimum_trades", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 77f8f2005..5dd587559 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -704,9 +704,10 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=20, ), - "overwrite_existing_exportfilename_content": Arg( - '--overwrite-existing-exportfilename-content', - help='overwrites existing contents if existent with exportfilename given', - action='store_true' - ) + "lookahead_analysis_exportfilename": Arg( + '--lookahead-analysis-exportfilename', + help="Use this filename to store lookahead-analysis-results", + default=None, + type=str + ), } diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 765f2caf2..78ad140de 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -175,8 +175,8 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: if lookaheadAnalysis_instances: LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookaheadAnalysis_instances) - if args['exportfilename'] is not None: - LookaheadAnalysisSubFunctions.export_to_csv(args, lookaheadAnalysis_instances) + if config['lookahead_analysis_exportfilename'] is not None: + LookaheadAnalysisSubFunctions.export_to_csv(config, lookaheadAnalysis_instances) else: logger.error("There were no strategies specified neither through " "--strategy nor through " diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 8e9a7fd7c..defb76b4b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -203,7 +203,7 @@ class Configuration: # This will override the strategy configuration self._args_to_config(config, argname='timeframe', logstring='Parameter -i/--timeframe detected ... ' - 'Using timeframe: {} ...') + 'Using timeframe: {} ...') self._args_to_config(config, argname='position_stacking', logstring='Parameter --enable-position-stacking detected ...') @@ -300,6 +300,10 @@ class Configuration: self._args_to_config(config, argname='hyperoptexportfilename', logstring='Using hyperopt file: {}') + if self.args["lookahead_analysis_exportfilename"] is not None: + self._args_to_config(config, argname='lookahead_analysis_exportfilename', + logstring='saving lookahead analysis results into {} ...') + self._args_to_config(config, argname='epochs', logstring='Parameter --epochs detected ... ' 'Will run Hyperopt with for {} epochs ...' diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 3ca1b71a0..8e6771b4b 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -296,7 +296,7 @@ class LookaheadAnalysisSubFunctions: print(table) @staticmethod - def export_to_csv(args: Dict[str, Any], lookahead_analysis: List[LookaheadAnalysis]): + def export_to_csv(config: Dict[str, Any], lookahead_analysis: List[LookaheadAnalysis]): def add_or_update_row(df, row_data): if ( (df['filename'] == row_data['filename']) & @@ -314,9 +314,9 @@ class LookaheadAnalysisSubFunctions: return df - if Path(args['exportfilename']).exists(): + if Path(config['lookahead_analysis_exportfilename']).exists(): # Read CSV file into a pandas dataframe - csv_df = pd.read_csv(args['exportfilename']) + csv_df = pd.read_csv(config['lookahead_analysis_exportfilename']) else: # Create a new empty DataFrame with the desired column names and set the index csv_df = pd.DataFrame(columns=[ @@ -335,8 +335,8 @@ class LookaheadAnalysisSubFunctions: 'biased_indicators': ",".join(inst.current_analysis.false_indicators)} csv_df = add_or_update_row(csv_df, new_row_data) - logger.info(f"saving {args['exportfilename']}") - csv_df.to_csv(args['exportfilename'], index=False) + logger.info(f"saving {config['lookahead_analysis_exportfilename']}") + csv_df.to_csv(config['lookahead_analysis_exportfilename'], index=False) @staticmethod def initialize_single_lookahead_analysis(strategy_obj: Dict[str, Any], config: Dict[str, Any], From 7d871faf04d5c03fa9aa213db93934e74f9576c1 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sat, 13 May 2023 22:40:11 +0200 Subject: [PATCH 09/55] added exportfilename to args_to_config introduced strategy_test_v3_with_lookahead_bias.py for checking lookahead_bias# introduced test_lookahead_analysis which currently is broken --- freqtrade/configuration/configuration.py | 7 +- .../strategy_test_v3_with_lookahead_bias.py | 50 ++++++++++++++ tests/test_lookahead_analysis.py | 69 +++++++++++++++++++ 3 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 tests/strategy/strats/strategy_test_v3_with_lookahead_bias.py create mode 100644 tests/test_lookahead_analysis.py diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index defb76b4b..c763d791a 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -300,9 +300,10 @@ class Configuration: self._args_to_config(config, argname='hyperoptexportfilename', logstring='Using hyperopt file: {}') - if self.args["lookahead_analysis_exportfilename"] is not None: - self._args_to_config(config, argname='lookahead_analysis_exportfilename', - logstring='saving lookahead analysis results into {} ...') + if self.args.get('lookahead_analysis_exportfilename'): + if self.args["lookahead_analysis_exportfilename"] is not None: + self._args_to_config(config, argname='lookahead_analysis_exportfilename', + logstring='saving lookahead analysis results into {} ...') self._args_to_config(config, argname='epochs', logstring='Parameter --epochs detected ... ' diff --git a/tests/strategy/strats/strategy_test_v3_with_lookahead_bias.py b/tests/strategy/strats/strategy_test_v3_with_lookahead_bias.py new file mode 100644 index 000000000..6cf894586 --- /dev/null +++ b/tests/strategy/strats/strategy_test_v3_with_lookahead_bias.py @@ -0,0 +1,50 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement +from pandas import DataFrame +from technical.indicators import ichimoku + +from freqtrade.strategy import IStrategy + + +class strategy_test_v3_with_lookahead_bias(IStrategy): + INTERFACE_VERSION = 3 + + # Minimal ROI designed for the strategy + minimal_roi = { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 + } + + # Optimal stoploss designed for the strategy + stoploss = -0.10 + + # Optimal timeframe for the strategy + timeframe = '5m' + + # Number of candles the strategy requires before producing valid signals + startup_candle_count: int = 20 + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # bias is introduced here + ichi = ichimoku(dataframe, + conversion_line_period=20, + base_line_periods=60, + laggin_span=120, + displacement=30) + dataframe['chikou_span'] = ichi['chikou_span'] + + return dataframe + + def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe.loc[ + dataframe['close'].shift(-10) > dataframe['close'], + 'enter_long'] = 1 + + return dataframe + + def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe.loc[ + dataframe['close'].shift(-10) > dataframe['close'], 'exit'] = 1 + + return dataframe diff --git a/tests/test_lookahead_analysis.py b/tests/test_lookahead_analysis.py new file mode 100644 index 000000000..24290798b --- /dev/null +++ b/tests/test_lookahead_analysis.py @@ -0,0 +1,69 @@ +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument + +from unittest.mock import PropertyMock + +import numpy as np + +import freqtrade.commands.arguments +import freqtrade.optimize.lookahead_analysis +from freqtrade.configuration import TimeRange +from freqtrade.data import history +from freqtrade.data.converter import clean_ohlcv_dataframe +from freqtrade.data.history import get_timerange +from tests.conftest import patch_exchange + + +def trim_dictlist(dict_list, num): + new = {} + for pair, pair_data in dict_list.items(): + new[pair] = pair_data[num:].reset_index() + return new + + +def load_data_test(what, testdatadir): + timerange = TimeRange.parse_timerange('1510694220-1510700340') + data = history.load_pair_history(pair='UNITTEST/BTC', datadir=testdatadir, + timeframe='1m', timerange=timerange, + drop_incomplete=False, + fill_up_missing=False) + + base = 0.001 + if what == 'raise': + data.loc[:, 'open'] = data.index * base + data.loc[:, 'high'] = data.index * base + 0.0001 + data.loc[:, 'low'] = data.index * base - 0.0001 + data.loc[:, 'close'] = data.index * base + + if what == 'lower': + data.loc[:, 'open'] = 1 - data.index * base + data.loc[:, 'high'] = 1 - data.index * base + 0.0001 + data.loc[:, 'low'] = 1 - data.index * base - 0.0001 + data.loc[:, 'close'] = 1 - data.index * base + + if what == 'sine': + hz = 0.1 # frequency + data.loc[:, 'open'] = np.sin(data.index * hz) / 1000 + base + data.loc[:, 'high'] = np.sin(data.index * hz) / 1000 + base + 0.0001 + data.loc[:, 'low'] = np.sin(data.index * hz) / 1000 + base - 0.0001 + data.loc[:, 'close'] = np.sin(data.index * hz) / 1000 + base + + return {'UNITTEST/BTC': clean_ohlcv_dataframe(data, timeframe='1m', pair='UNITTEST/BTC', + fill_missing=True, drop_incomplete=True)} + + +def test_biased_strategy(default_conf, mocker, caplog) -> None: + + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + patch_exchange(mocker) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + + default_conf['timeframe'] = '5m' + default_conf['timerange'] = '-1510694220' + default_conf['strategy'] = 'strategy_test_v3_with_lookahead_bias' + default_conf['strategy_path'] = 'tests/strategy/strats' + + strategy_obj = {} + strategy_obj['name'] = "strategy_test_v3_with_lookahead_bias" + freqtrade.optimize.lookahead_analysis.LookaheadAnalysis(default_conf, strategy_obj, {}) + pass From 5488789bc467e262e23116df5480691a2c064330 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:01:42 +0200 Subject: [PATCH 10/55] Arguments should be in the configuration. --- freqtrade/commands/cli_options.py | 2 -- freqtrade/constants.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 5dd587559..33de428db 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -695,14 +695,12 @@ AVAILABLE_CLI_OPTIONS = { help='set INT minimum trade amount', type=check_int_positive, metavar='INT', - default=10, ), "targeted_trade_amount": Arg( '--targeted-trade-amount', help='set INT targeted trade amount', type=check_int_positive, metavar='INT', - default=20, ), "lookahead_analysis_exportfilename": Arg( '--lookahead-analysis-exportfilename', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index b8e240419..ef59d8999 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -164,6 +164,8 @@ CONF_SCHEMA = { 'trading_mode': {'type': 'string', 'enum': TRADING_MODES}, 'margin_mode': {'type': 'string', 'enum': MARGIN_MODES}, 'reduce_df_footprint': {'type': 'boolean', 'default': False}, + 'minimum_trade_amount': {'type': 'number', 'default': 10}, + 'targeted_trade_amount': {'type': 'number', 'default': 20}, 'liquidation_buffer': {'type': 'number', 'minimum': 0.0, 'maximum': 0.99}, 'backtest_breakdown': { 'type': 'array', From 2e79aaae0023635d24ec6016978dcf6aca113d9d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:02:13 +0200 Subject: [PATCH 11/55] Remove usage of args. It's clumsy to use and prevents specifying settings in the configuration. --- freqtrade/commands/optimize_commands.py | 11 ++++------- freqtrade/optimize/lookahead_analysis.py | 13 ++++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 78ad140de..866bf8e61 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -144,7 +144,7 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - if args['targeted_trade_amount'] < args['minimum_trade_amount']: + if config['targeted_trade_amount'] < config['minimum_trade_amount']: # add logic that tells the user to check the configuration # since this combo doesn't make any sense. pass @@ -153,13 +153,10 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) lookaheadAnalysis_instances = [] - strategy_list = [] # unify --strategy and --strategy_list to one list - if 'strategy' in args and args['strategy'] is not None: - strategy_list = [args['strategy']] - else: - strategy_list = args['strategy_list'] + if not (strategy_list := config.get('strategy_list', [])): + strategy_list = [config['strategy']] # check if strategies can be properly loaded, only check them if they can be. if strategy_list is not None: @@ -168,7 +165,7 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: if strategy_obj['name'] == strat and strategy_obj not in strategy_list: lookaheadAnalysis_instances.append( LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( - strategy_obj, config, args)) + strategy_obj, config)) break # report the results diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 8e6771b4b..90aa934a6 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -42,7 +42,7 @@ class Analysis: class LookaheadAnalysis: - def __init__(self, config: Dict[str, Any], strategy_obj: dict, args: Dict[str, Any]): + def __init__(self, config: Dict[str, Any], strategy_obj: Dict): self.failed_bias_check = True self.full_varHolder = VarHolder @@ -53,9 +53,9 @@ class LookaheadAnalysis: self.local_config = deepcopy(config) self.local_config['strategy'] = strategy_obj['name'] self.current_analysis = Analysis() - self.minimum_trade_amount = args['minimum_trade_amount'] - self.targeted_trade_amount = args['targeted_trade_amount'] - self.exportfilename = args['exportfilename'] + self.minimum_trade_amount = config['minimum_trade_amount'] + self.targeted_trade_amount = config['targeted_trade_amount'] + self.exportfilename = config['exportfilename'] self.strategy_obj = strategy_obj @staticmethod @@ -339,12 +339,11 @@ class LookaheadAnalysisSubFunctions: csv_df.to_csv(config['lookahead_analysis_exportfilename'], index=False) @staticmethod - def initialize_single_lookahead_analysis(strategy_obj: Dict[str, Any], config: Dict[str, Any], - args: Dict[str, Any]): + def initialize_single_lookahead_analysis(strategy_obj: Dict[str, Any], config: Dict[str, Any]): logger.info(f"Bias test of {Path(strategy_obj['location']).name} started.") start = time.perf_counter() - current_instance = LookaheadAnalysis(config, strategy_obj, args) + current_instance = LookaheadAnalysis(config, strategy_obj) current_instance.start() elapsed = time.perf_counter() - start logger.info(f"checking look ahead bias via backtests " From a0edbe4797e7edbdf287d86a61c113bbbf3bea0d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:06:50 +0200 Subject: [PATCH 12/55] Switch to using config instead of args. --- freqtrade/commands/cli_options.py | 5 ++--- freqtrade/commands/optimize_commands.py | 2 +- freqtrade/configuration/configuration.py | 13 +++++++++++++ freqtrade/constants.py | 1 + 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 33de428db..e4a864ea0 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -692,20 +692,19 @@ AVAILABLE_CLI_OPTIONS = { ), "minimum_trade_amount": Arg( '--minimum-trade-amount', - help='set INT minimum trade amount', + help='Minimum trade amount for lookahead-analysis', type=check_int_positive, metavar='INT', ), "targeted_trade_amount": Arg( '--targeted-trade-amount', - help='set INT targeted trade amount', + help='Targeted trade amount for lookahead analysis', type=check_int_positive, metavar='INT', ), "lookahead_analysis_exportfilename": Arg( '--lookahead-analysis-exportfilename', help="Use this filename to store lookahead-analysis-results", - default=None, type=str ), } diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 866bf8e61..d5d4a0625 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -172,7 +172,7 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: if lookaheadAnalysis_instances: LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookaheadAnalysis_instances) - if config['lookahead_analysis_exportfilename'] is not None: + if config.get('lookahead_analysis_exportfilename') is not None: LookaheadAnalysisSubFunctions.export_to_csv(config, lookaheadAnalysis_instances) else: logger.error("There were no strategies specified neither through " diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index c763d791a..5bbbf301d 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -479,6 +479,19 @@ class Configuration: self._args_to_config(config, argname='analysis_csv_path', logstring='Path to store analysis CSVs: {}') + self._args_to_config(config, argname='analysis_csv_path', + logstring='Path to store analysis CSVs: {}') + + # Lookahead analysis results + self._args_to_config(config, argname='targeted_trade_amount', + logstring='Targeted Trade amount: {}') + + self._args_to_config(config, argname='minimum_trade_amount', + logstring='Minimum Trade amount: {}') + + self._args_to_config(config, argname='lookahead_analysis_exportfilename', + logstring='Path to store lookahead-analysis-results: {}') + def _process_runmode(self, config: Config) -> None: self._args_to_config(config, argname='dry_run', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index ef59d8999..30484e560 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -166,6 +166,7 @@ CONF_SCHEMA = { 'reduce_df_footprint': {'type': 'boolean', 'default': False}, 'minimum_trade_amount': {'type': 'number', 'default': 10}, 'targeted_trade_amount': {'type': 'number', 'default': 20}, + 'lookahead_analysis_exportfilename': {'type': 'string'}, 'liquidation_buffer': {'type': 'number', 'minimum': 0.0, 'maximum': 0.99}, 'backtest_breakdown': { 'type': 'array', From 073dac8d5f2956a2f04383fd420fb91b61b0546f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:08:22 +0200 Subject: [PATCH 13/55] Move lookahead analysis tests to optimize subdir --- tests/{ => optimize}/test_lookahead_analysis.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{ => optimize}/test_lookahead_analysis.py (100%) diff --git a/tests/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py similarity index 100% rename from tests/test_lookahead_analysis.py rename to tests/optimize/test_lookahead_analysis.py From 2e675efa13e86d7a3c73097bab50fe57d1ca4545 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:14:13 +0200 Subject: [PATCH 14/55] Initial fix - test --- freqtrade/optimize/lookahead_analysis.py | 1 - tests/optimize/test_lookahead_analysis.py | 53 +++++++---------------- 2 files changed, 15 insertions(+), 39 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 90aa934a6..4de9d755e 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -55,7 +55,6 @@ class LookaheadAnalysis: self.current_analysis = Analysis() self.minimum_trade_amount = config['minimum_trade_amount'] self.targeted_trade_amount = config['targeted_trade_amount'] - self.exportfilename = config['exportfilename'] self.strategy_obj = strategy_obj @staticmethod diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 24290798b..945b3893b 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -3,6 +3,7 @@ from unittest.mock import PropertyMock import numpy as np +import pytest import freqtrade.commands.arguments import freqtrade.optimize.lookahead_analysis @@ -10,7 +11,15 @@ from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.converter import clean_ohlcv_dataframe from freqtrade.data.history import get_timerange -from tests.conftest import patch_exchange +from tests.conftest import generate_test_data, patch_exchange + + +@pytest.fixture +def lookahead_conf(default_conf_usdt): + default_conf_usdt['minimum_trade_amount'] = 10 + default_conf_usdt['targeted_trade_amount'] = 20 + + return default_conf_usdt def trim_dictlist(dict_list, num): @@ -20,50 +29,18 @@ def trim_dictlist(dict_list, num): return new -def load_data_test(what, testdatadir): - timerange = TimeRange.parse_timerange('1510694220-1510700340') - data = history.load_pair_history(pair='UNITTEST/BTC', datadir=testdatadir, - timeframe='1m', timerange=timerange, - drop_incomplete=False, - fill_up_missing=False) - - base = 0.001 - if what == 'raise': - data.loc[:, 'open'] = data.index * base - data.loc[:, 'high'] = data.index * base + 0.0001 - data.loc[:, 'low'] = data.index * base - 0.0001 - data.loc[:, 'close'] = data.index * base - - if what == 'lower': - data.loc[:, 'open'] = 1 - data.index * base - data.loc[:, 'high'] = 1 - data.index * base + 0.0001 - data.loc[:, 'low'] = 1 - data.index * base - 0.0001 - data.loc[:, 'close'] = 1 - data.index * base - - if what == 'sine': - hz = 0.1 # frequency - data.loc[:, 'open'] = np.sin(data.index * hz) / 1000 + base - data.loc[:, 'high'] = np.sin(data.index * hz) / 1000 + base + 0.0001 - data.loc[:, 'low'] = np.sin(data.index * hz) / 1000 + base - 0.0001 - data.loc[:, 'close'] = np.sin(data.index * hz) / 1000 + base - - return {'UNITTEST/BTC': clean_ohlcv_dataframe(data, timeframe='1m', pair='UNITTEST/BTC', - fill_missing=True, drop_incomplete=True)} - - -def test_biased_strategy(default_conf, mocker, caplog) -> None: +def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: mocker.patch('freqtrade.data.history.get_timerange', get_timerange) patch_exchange(mocker) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) - default_conf['timeframe'] = '5m' - default_conf['timerange'] = '-1510694220' - default_conf['strategy'] = 'strategy_test_v3_with_lookahead_bias' - default_conf['strategy_path'] = 'tests/strategy/strats' + lookahead_conf['timeframe'] = '5m' + lookahead_conf['timerange'] = '-1510694220' + lookahead_conf['strategy'] = 'strategy_test_v3_with_lookahead_bias' strategy_obj = {} strategy_obj['name'] = "strategy_test_v3_with_lookahead_bias" - freqtrade.optimize.lookahead_analysis.LookaheadAnalysis(default_conf, strategy_obj, {}) + freqtrade.optimize.lookahead_analysis.LookaheadAnalysis(lookahead_conf, strategy_obj) pass From 209eb63edebe16810da8c6e9be4ecc0c09f74607 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:28:52 +0200 Subject: [PATCH 15/55] Add startup test case --- freqtrade/commands/optimize_commands.py | 7 ++- tests/optimize/test_lookahead_analysis.py | 56 ++++++++++++++++++----- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index d5d4a0625..fb2d5ff21 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -6,7 +6,6 @@ from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import round_coin_value -from freqtrade.optimize.lookahead_analysis import LookaheadAnalysisSubFunctions from freqtrade.resolvers import StrategyResolver @@ -142,12 +141,16 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ + from freqtrade.optimize.lookahead_analysis import LookaheadAnalysisSubFunctions + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if config['targeted_trade_amount'] < config['minimum_trade_amount']: # add logic that tells the user to check the configuration # since this combo doesn't make any sense. - pass + raise OperationalException( + "targeted trade amount can't be smaller than minimum trade amount." + ) strategy_objs = StrategyResolver.search_all_objects( config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 945b3893b..ff6d9c7da 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -1,17 +1,16 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument -from unittest.mock import PropertyMock +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock -import numpy as np import pytest import freqtrade.commands.arguments import freqtrade.optimize.lookahead_analysis -from freqtrade.configuration import TimeRange -from freqtrade.data import history -from freqtrade.data.converter import clean_ohlcv_dataframe +from freqtrade.commands.optimize_commands import start_lookahead_analysis from freqtrade.data.history import get_timerange -from tests.conftest import generate_test_data, patch_exchange +from freqtrade.exceptions import OperationalException +from tests.conftest import CURRENT_TEST_STRATEGY, get_args, patch_exchange @pytest.fixture @@ -22,11 +21,46 @@ def lookahead_conf(default_conf_usdt): return default_conf_usdt -def trim_dictlist(dict_list, num): - new = {} - for pair, pair_data in dict_list.items(): - new[pair] = pair_data[num:].reset_index() - return new +def test_start_start_lookahead_analysis(mocker): + single_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.lookahead_analysis.LookaheadAnalysisSubFunctions', + initialize_single_lookahead_analysis=single_mock, + text_table_lookahead_analysis_instances=MagicMock(), + ) + args = [ + "lookahead-analysis", + "--strategy", + CURRENT_TEST_STRATEGY, + "--strategy-path", + str(Path(__file__).parent.parent / "strategy" / "strats"), + ] + pargs = get_args(args) + pargs['config'] = None + + start_lookahead_analysis(pargs) + assert single_mock.call_count == 1 + + single_mock.reset_mock() + + # Test invalid config + args = [ + "lookahead-analysis", + "--strategy", + CURRENT_TEST_STRATEGY, + "--strategy-path", + str(Path(__file__).parent.parent / "strategy" / "strats"), + "--targeted-trade-amount", + "10", + "--minimum-trade-amount", + "20", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"targeted trade amount can't be smaller than .*"): + start_lookahead_analysis(pargs) + def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: From 7b9f82c71a960b7314ccffea1d2c6c23e9872da8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:30:51 +0200 Subject: [PATCH 16/55] Remove needless check for "None" list --- freqtrade/commands/optimize_commands.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index fb2d5ff21..9c753dcb9 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -162,14 +162,13 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: strategy_list = [config['strategy']] # check if strategies can be properly loaded, only check them if they can be. - if strategy_list is not None: - for strat in strategy_list: - for strategy_obj in strategy_objs: - if strategy_obj['name'] == strat and strategy_obj not in strategy_list: - lookaheadAnalysis_instances.append( - LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( - strategy_obj, config)) - break + for strat in strategy_list: + for strategy_obj in strategy_objs: + if strategy_obj['name'] == strat and strategy_obj not in strategy_list: + lookaheadAnalysis_instances.append( + LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( + strategy_obj, config)) + break # report the results if lookaheadAnalysis_instances: From 1c4a7c7a05a7619e24b2c9af64518d999379e42b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:35:17 +0200 Subject: [PATCH 17/55] Split Lookahead helper to separate file --- freqtrade/commands/optimize_commands.py | 2 +- freqtrade/optimize/lookahead_analysis.py | 87 ----------------- .../optimize/lookahead_analysis_helpers.py | 95 +++++++++++++++++++ 3 files changed, 96 insertions(+), 88 deletions(-) create mode 100644 freqtrade/optimize/lookahead_analysis_helpers.py diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 9c753dcb9..06e9b7adb 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -141,7 +141,7 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ - from freqtrade.optimize.lookahead_analysis import LookaheadAnalysisSubFunctions + from freqtrade.optimize.lookahead_analysis_helpers import LookaheadAnalysisSubFunctions config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 4de9d755e..e8631e8b6 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -2,10 +2,8 @@ import copy import logging import pathlib import shutil -import time from copy import deepcopy from datetime import datetime, timedelta, timezone -from pathlib import Path from typing import Any, Dict, List import pandas as pd @@ -263,89 +261,4 @@ class LookaheadAnalysis: self.failed_bias_check = False -class LookaheadAnalysisSubFunctions: - @staticmethod - def text_table_lookahead_analysis_instances(lookahead_instances: List[LookaheadAnalysis]): - headers = ['filename', 'strategy', 'has_bias', 'total_signals', - 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators'] - data = [] - for inst in lookahead_instances: - if inst.failed_bias_check: - data.append( - [ - inst.strategy_obj['location'].parts[-1], - inst.strategy_obj['name'], - 'error while checking' - ] - ) - else: - data.append( - [ - inst.strategy_obj['location'].parts[-1], - inst.strategy_obj['name'], - inst.current_analysis.has_bias, - inst.current_analysis.total_signals, - inst.current_analysis.false_entry_signals, - inst.current_analysis.false_exit_signals, - ", ".join(inst.current_analysis.false_indicators) - ] - ) - from tabulate import tabulate - table = tabulate(data, headers=headers, tablefmt="orgtbl") - print(table) - - @staticmethod - def export_to_csv(config: Dict[str, Any], lookahead_analysis: List[LookaheadAnalysis]): - def add_or_update_row(df, row_data): - if ( - (df['filename'] == row_data['filename']) & - (df['strategy'] == row_data['strategy']) - ).any(): - # Update existing row - pd_series = pd.DataFrame([row_data]) - df.loc[ - (df['filename'] == row_data['filename']) & - (df['strategy'] == row_data['strategy']) - ] = pd_series - else: - # Add new row - df = pd.concat([df, pd.DataFrame([row_data], columns=df.columns)]) - return df - - if Path(config['lookahead_analysis_exportfilename']).exists(): - # Read CSV file into a pandas dataframe - csv_df = pd.read_csv(config['lookahead_analysis_exportfilename']) - else: - # Create a new empty DataFrame with the desired column names and set the index - csv_df = pd.DataFrame(columns=[ - 'filename', 'strategy', 'has_bias', 'total_signals', - 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators' - ], - index=None) - - for inst in lookahead_analysis: - new_row_data = {'filename': inst.strategy_obj['location'].parts[-1], - 'strategy': inst.strategy_obj['name'], - 'has_bias': inst.current_analysis.has_bias, - 'total_signals': inst.current_analysis.total_signals, - 'biased_entry_signals': inst.current_analysis.false_entry_signals, - 'biased_exit_signals': inst.current_analysis.false_exit_signals, - 'biased_indicators': ",".join(inst.current_analysis.false_indicators)} - csv_df = add_or_update_row(csv_df, new_row_data) - - logger.info(f"saving {config['lookahead_analysis_exportfilename']}") - csv_df.to_csv(config['lookahead_analysis_exportfilename'], index=False) - - @staticmethod - def initialize_single_lookahead_analysis(strategy_obj: Dict[str, Any], config: Dict[str, Any]): - - logger.info(f"Bias test of {Path(strategy_obj['location']).name} started.") - start = time.perf_counter() - current_instance = LookaheadAnalysis(config, strategy_obj) - current_instance.start() - elapsed = time.perf_counter() - start - logger.info(f"checking look ahead bias via backtests " - f"of {Path(strategy_obj['location']).name} " - f"took {elapsed:.0f} seconds.") - return current_instance diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py new file mode 100644 index 000000000..987a55b24 --- /dev/null +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -0,0 +1,95 @@ +import time +from pathlib import Path +from typing import Any, Dict, List + +import pandas as pd + +from freqtrade.optimize.lookahead_analysis import LookaheadAnalysis, logger + + +class LookaheadAnalysisSubFunctions: + @staticmethod + def text_table_lookahead_analysis_instances(lookahead_instances: List[LookaheadAnalysis]): + headers = ['filename', 'strategy', 'has_bias', 'total_signals', + 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators'] + data = [] + for inst in lookahead_instances: + if inst.failed_bias_check: + data.append( + [ + inst.strategy_obj['location'].parts[-1], + inst.strategy_obj['name'], + 'error while checking' + ] + ) + else: + data.append( + [ + inst.strategy_obj['location'].parts[-1], + inst.strategy_obj['name'], + inst.current_analysis.has_bias, + inst.current_analysis.total_signals, + inst.current_analysis.false_entry_signals, + inst.current_analysis.false_exit_signals, + ", ".join(inst.current_analysis.false_indicators) + ] + ) + from tabulate import tabulate + table = tabulate(data, headers=headers, tablefmt="orgtbl") + print(table) + + @staticmethod + def export_to_csv(config: Dict[str, Any], lookahead_analysis: List[LookaheadAnalysis]): + def add_or_update_row(df, row_data): + if ( + (df['filename'] == row_data['filename']) & + (df['strategy'] == row_data['strategy']) + ).any(): + # Update existing row + pd_series = pd.DataFrame([row_data]) + df.loc[ + (df['filename'] == row_data['filename']) & + (df['strategy'] == row_data['strategy']) + ] = pd_series + else: + # Add new row + df = pd.concat([df, pd.DataFrame([row_data], columns=df.columns)]) + + return df + + if Path(config['lookahead_analysis_exportfilename']).exists(): + # Read CSV file into a pandas dataframe + csv_df = pd.read_csv(config['lookahead_analysis_exportfilename']) + else: + # Create a new empty DataFrame with the desired column names and set the index + csv_df = pd.DataFrame(columns=[ + 'filename', 'strategy', 'has_bias', 'total_signals', + 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators' + ], + index=None) + + for inst in lookahead_analysis: + new_row_data = {'filename': inst.strategy_obj['location'].parts[-1], + 'strategy': inst.strategy_obj['name'], + 'has_bias': inst.current_analysis.has_bias, + 'total_signals': inst.current_analysis.total_signals, + 'biased_entry_signals': inst.current_analysis.false_entry_signals, + 'biased_exit_signals': inst.current_analysis.false_exit_signals, + 'biased_indicators': ",".join(inst.current_analysis.false_indicators)} + csv_df = add_or_update_row(csv_df, new_row_data) + + logger.info(f"saving {config['lookahead_analysis_exportfilename']}") + csv_df.to_csv(config['lookahead_analysis_exportfilename'], index=False) + + @staticmethod + def initialize_single_lookahead_analysis(strategy_obj: Dict[str, Any], config: Dict[str, Any]): + + logger.info(f"Bias test of {Path(strategy_obj['location']).name} started.") + start = time.perf_counter() + current_instance = LookaheadAnalysis(config, strategy_obj) + current_instance.start() + elapsed = time.perf_counter() - start + logger.info(f"checking look ahead bias via backtests " + f"of {Path(strategy_obj['location']).name} " + f"took {elapsed:.0f} seconds.") + return current_instance From d8af0dc9c4e1c2dad1844d611e78d3b1b86b997b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:36:58 +0200 Subject: [PATCH 18/55] Slightly improve testcase --- tests/optimize/test_lookahead_analysis.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index ff6d9c7da..6872cb73a 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -5,11 +5,10 @@ from unittest.mock import MagicMock, PropertyMock import pytest -import freqtrade.commands.arguments -import freqtrade.optimize.lookahead_analysis from freqtrade.commands.optimize_commands import start_lookahead_analysis from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException +from freqtrade.optimize.lookahead_analysis import LookaheadAnalysis from tests.conftest import CURRENT_TEST_STRATEGY, get_args, patch_exchange @@ -21,12 +20,13 @@ def lookahead_conf(default_conf_usdt): return default_conf_usdt -def test_start_start_lookahead_analysis(mocker): +def test_start_lookahead_analysis(mocker): single_mock = MagicMock() + text_table_mock = MagicMock() mocker.patch.multiple( - 'freqtrade.optimize.lookahead_analysis.LookaheadAnalysisSubFunctions', + 'freqtrade.optimize.lookahead_analysis_helpers.LookaheadAnalysisSubFunctions', initialize_single_lookahead_analysis=single_mock, - text_table_lookahead_analysis_instances=MagicMock(), + text_table_lookahead_analysis_instances=text_table_mock, ) args = [ "lookahead-analysis", @@ -40,6 +40,7 @@ def test_start_start_lookahead_analysis(mocker): start_lookahead_analysis(pargs) assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 single_mock.reset_mock() @@ -62,7 +63,6 @@ def test_start_start_lookahead_analysis(mocker): start_lookahead_analysis(pargs) - def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: mocker.patch('freqtrade.data.history.get_timerange', get_timerange) @@ -76,5 +76,5 @@ def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: strategy_obj = {} strategy_obj['name'] = "strategy_test_v3_with_lookahead_bias" - freqtrade.optimize.lookahead_analysis.LookaheadAnalysis(lookahead_conf, strategy_obj) + LookaheadAnalysis(lookahead_conf, strategy_obj) pass From ceddcd9242209211153e30c728df77f9e907b4a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:45:06 +0200 Subject: [PATCH 19/55] Move most of the logic to lookahead_analysis helper --- freqtrade/commands/optimize_commands.py | 39 +-------------- .../optimize/lookahead_analysis_helpers.py | 47 ++++++++++++++++++- tests/optimize/test_lookahead_analysis.py | 1 + 3 files changed, 49 insertions(+), 38 deletions(-) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 06e9b7adb..4b8763737 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -6,7 +6,6 @@ from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import round_coin_value -from freqtrade.resolvers import StrategyResolver logger = logging.getLogger(__name__) @@ -144,40 +143,6 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: from freqtrade.optimize.lookahead_analysis_helpers import LookaheadAnalysisSubFunctions config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + LookaheadAnalysisSubFunctions.start(config) + - if config['targeted_trade_amount'] < config['minimum_trade_amount']: - # add logic that tells the user to check the configuration - # since this combo doesn't make any sense. - raise OperationalException( - "targeted trade amount can't be smaller than minimum trade amount." - ) - - strategy_objs = StrategyResolver.search_all_objects( - config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) - - lookaheadAnalysis_instances = [] - - # unify --strategy and --strategy_list to one list - if not (strategy_list := config.get('strategy_list', [])): - strategy_list = [config['strategy']] - - # check if strategies can be properly loaded, only check them if they can be. - for strat in strategy_list: - for strategy_obj in strategy_objs: - if strategy_obj['name'] == strat and strategy_obj not in strategy_list: - lookaheadAnalysis_instances.append( - LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( - strategy_obj, config)) - break - - # report the results - if lookaheadAnalysis_instances: - LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( - lookaheadAnalysis_instances) - if config.get('lookahead_analysis_exportfilename') is not None: - LookaheadAnalysisSubFunctions.export_to_csv(config, lookaheadAnalysis_instances) - else: - logger.error("There were no strategies specified neither through " - "--strategy nor through " - "--strategy_list " - "or timeframe was not specified.") diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 987a55b24..e2e9ffb42 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -1,10 +1,17 @@ +import logging import time from pathlib import Path from typing import Any, Dict, List import pandas as pd -from freqtrade.optimize.lookahead_analysis import LookaheadAnalysis, logger +from freqtrade.constants import Config +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.lookahead_analysis import LookaheadAnalysis +from freqtrade.resolvers import StrategyResolver + + +logger = logging.getLogger(__name__) class LookaheadAnalysisSubFunctions: @@ -93,3 +100,41 @@ class LookaheadAnalysisSubFunctions: f"of {Path(strategy_obj['location']).name} " f"took {elapsed:.0f} seconds.") return current_instance + + @staticmethod + def start(config: Config): + if config['targeted_trade_amount'] < config['minimum_trade_amount']: + # this combo doesn't make any sense. + raise OperationalException( + "targeted trade amount can't be smaller than minimum trade amount." + ) + + strategy_objs = StrategyResolver.search_all_objects( + config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) + + lookaheadAnalysis_instances = [] + + # unify --strategy and --strategy_list to one list + if not (strategy_list := config.get('strategy_list', [])): + strategy_list = [config['strategy']] + + # check if strategies can be properly loaded, only check them if they can be. + for strat in strategy_list: + for strategy_obj in strategy_objs: + if strategy_obj['name'] == strat and strategy_obj not in strategy_list: + lookaheadAnalysis_instances.append( + LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( + strategy_obj, config)) + break + + # report the results + if lookaheadAnalysis_instances: + LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + lookaheadAnalysis_instances) + if config.get('lookahead_analysis_exportfilename') is not None: + LookaheadAnalysisSubFunctions.export_to_csv(config, lookaheadAnalysis_instances) + else: + logger.error("There were no strategies specified neither through " + "--strategy nor through " + "--strategy_list " + "or timeframe was not specified.") diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 6872cb73a..3136d6a16 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -77,4 +77,5 @@ def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: strategy_obj = {} strategy_obj['name'] = "strategy_test_v3_with_lookahead_bias" LookaheadAnalysis(lookahead_conf, strategy_obj) + pass From e183707979b5fb9ec94eb022c82de8f7a2a170ef Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 11:51:46 +0200 Subject: [PATCH 20/55] Further test lookahead_helpers --- .../optimize/lookahead_analysis_helpers.py | 5 +++ tests/optimize/test_lookahead_analysis.py | 35 ++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index e2e9ffb42..53d8c8a59 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -116,6 +116,11 @@ class LookaheadAnalysisSubFunctions: # unify --strategy and --strategy_list to one list if not (strategy_list := config.get('strategy_list', [])): + if config.get('strategy') is None: + raise OperationalException( + "No Strategy specified. Please specify a strategy via --strategy or " + "--strategy_list" + ) strategy_list = [config['strategy']] # check if strategies can be properly loaded, only check them if they can be. diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 3136d6a16..2eb0b2657 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument +from copy import deepcopy from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -9,7 +10,8 @@ from freqtrade.commands.optimize_commands import start_lookahead_analysis from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException from freqtrade.optimize.lookahead_analysis import LookaheadAnalysis -from tests.conftest import CURRENT_TEST_STRATEGY, get_args, patch_exchange +from freqtrade.optimize.lookahead_analysis_helpers import LookaheadAnalysisSubFunctions +from tests.conftest import CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange @pytest.fixture @@ -63,6 +65,37 @@ def test_start_lookahead_analysis(mocker): start_lookahead_analysis(pargs) +def test_lookahead_helper_invalid_config(lookahead_conf, mocker, caplog) -> None: + conf = deepcopy(lookahead_conf) + conf['targeted_trade_amount'] = 10 + conf['minimum_trade_amount'] = 40 + with pytest.raises(OperationalException, + match=r"targeted trade amount can't be smaller than .*"): + LookaheadAnalysisSubFunctions.start(conf) + + conf = deepcopy(lookahead_conf) + del conf['strategy'] + with pytest.raises(OperationalException, + match=r"No Strategy specified"): + LookaheadAnalysisSubFunctions.start(conf) + + +def test_lookahead_helper_start(lookahead_conf, mocker, caplog) -> None: + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.lookahead_analysis_helpers.LookaheadAnalysisSubFunctions', + initialize_single_lookahead_analysis=single_mock, + text_table_lookahead_analysis_instances=text_table_mock, + ) + LookaheadAnalysisSubFunctions.start(lookahead_conf) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 + + single_mock.reset_mock() + text_table_mock.reset_mock() + + def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: mocker.patch('freqtrade.data.history.get_timerange', get_timerange) From 3f5c18a035a311dd01fb60d5040a8c0b5689b3df Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 12:03:06 +0200 Subject: [PATCH 21/55] Add some tests as todo --- tests/optimize/test_lookahead_analysis.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 2eb0b2657..0092b6074 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -96,6 +96,21 @@ def test_lookahead_helper_start(lookahead_conf, mocker, caplog) -> None: text_table_mock.reset_mock() +def test_lookahead_helper_text_table_lookahead_analysis_instances(): + # TODO + pytest.skip("TODO") + + +def test_lookahead_helper_export_to_csv(): + # TODO + pytest.skip("TODO") + + +def test_initialize_single_lookahead_analysis(): + # TODO + pytest.skip("TODO") + + def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: mocker.patch('freqtrade.data.history.get_timerange', get_timerange) From 9869a21951c4947f6d1f070ff40c432e78515a36 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 15:39:21 +0200 Subject: [PATCH 22/55] Move strategy to it's own directory to avoid having other --- freqtrade/optimize/lookahead_analysis.py | 3 --- tests/optimize/test_lookahead_analysis.py | 19 +++++++++++++------ .../strategy_test_v3_with_lookahead_bias.py | 0 3 files changed, 13 insertions(+), 9 deletions(-) rename tests/strategy/strats/{ => lookahead_bias}/strategy_test_v3_with_lookahead_bias.py (100%) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index e8631e8b6..dfb1a9ea2 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -259,6 +259,3 @@ class LookaheadAnalysis: logging.info(self.local_config['strategy'] + ": no bias detected") self.failed_bias_check = False - - - diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 0092b6074..944892685 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -11,13 +11,15 @@ from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException from freqtrade.optimize.lookahead_analysis import LookaheadAnalysis from freqtrade.optimize.lookahead_analysis_helpers import LookaheadAnalysisSubFunctions -from tests.conftest import CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange +from tests.conftest import EXMS, get_args, patch_exchange @pytest.fixture def lookahead_conf(default_conf_usdt): default_conf_usdt['minimum_trade_amount'] = 10 default_conf_usdt['targeted_trade_amount'] = 20 + default_conf_usdt['strategy_path'] = str( + Path(__file__).parent.parent / "strategy/strats/lookahead_bias") return default_conf_usdt @@ -33,7 +35,7 @@ def test_start_lookahead_analysis(mocker): args = [ "lookahead-analysis", "--strategy", - CURRENT_TEST_STRATEGY, + "strategy_test_v3_with_lookahead_bias", "--strategy-path", str(Path(__file__).parent.parent / "strategy" / "strats"), ] @@ -50,7 +52,7 @@ def test_start_lookahead_analysis(mocker): args = [ "lookahead-analysis", "--strategy", - CURRENT_TEST_STRATEGY, + "strategy_test_v3_with_lookahead_bias", "--strategy-path", str(Path(__file__).parent.parent / "strategy" / "strats"), "--targeted-trade-amount", @@ -114,16 +116,21 @@ def test_initialize_single_lookahead_analysis(): def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + mocker.patch(f'{EXMS}.get_fee', return_value=0.0) + mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) + mocker.patch(f'{EXMS}.get_max_pair_stake_amount', return_value=float('inf')) patch_exchange(mocker) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) + lookahead_conf['pairs'] = ['UNITTEST/USDT'] lookahead_conf['timeframe'] = '5m' - lookahead_conf['timerange'] = '-1510694220' + lookahead_conf['timerange'] = '1516406400-1517270400' lookahead_conf['strategy'] = 'strategy_test_v3_with_lookahead_bias' strategy_obj = {} strategy_obj['name'] = "strategy_test_v3_with_lookahead_bias" - LookaheadAnalysis(lookahead_conf, strategy_obj) + instance = LookaheadAnalysis(lookahead_conf, strategy_obj) + instance.start() - pass + # TODO: assert something ... most likely output (?) or instance state? diff --git a/tests/strategy/strats/strategy_test_v3_with_lookahead_bias.py b/tests/strategy/strats/lookahead_bias/strategy_test_v3_with_lookahead_bias.py similarity index 100% rename from tests/strategy/strats/strategy_test_v3_with_lookahead_bias.py rename to tests/strategy/strats/lookahead_bias/strategy_test_v3_with_lookahead_bias.py From e73cd1487e61825614206ed4019b79e41aea9795 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 19:57:26 +0200 Subject: [PATCH 23/55] Add somewhat sensible assert --- tests/optimize/test_lookahead_analysis.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 944892685..ef4f8809c 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -11,7 +11,7 @@ from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException from freqtrade.optimize.lookahead_analysis import LookaheadAnalysis from freqtrade.optimize.lookahead_analysis_helpers import LookaheadAnalysisSubFunctions -from tests.conftest import EXMS, get_args, patch_exchange +from tests.conftest import EXMS, get_args, log_has_re, patch_exchange @pytest.fixture @@ -125,12 +125,13 @@ def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: lookahead_conf['pairs'] = ['UNITTEST/USDT'] lookahead_conf['timeframe'] = '5m' - lookahead_conf['timerange'] = '1516406400-1517270400' + lookahead_conf['timerange'] = '20180119-20180122' lookahead_conf['strategy'] = 'strategy_test_v3_with_lookahead_bias' strategy_obj = {} strategy_obj['name'] = "strategy_test_v3_with_lookahead_bias" instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.start() + assert log_has_re(r".*bias detected.*", caplog) # TODO: assert something ... most likely output (?) or instance state? From 104fa9e32db3dc69d5fdf43ffffdcc03dbd58d5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 19:58:14 +0200 Subject: [PATCH 24/55] Use logger, not the logging module --- freqtrade/optimize/lookahead_analysis.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index dfb1a9ea2..e40322c88 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -121,9 +121,9 @@ class LookaheadAnalysis: if not self.current_analysis.false_indicators.__contains__(col_name[0]): self.current_analysis.false_indicators.append(col_name[0]) - logging.info(f"=> found look ahead bias in indicator " - f"{col_name[0]}. " - f"{str(self_value)} != {str(other_value)}") + logger.info(f"=> found look ahead bias in indicator " + f"{col_name[0]}. " + f"{str(self_value)} != {str(other_value)}") def prepare_data(self, varholder: VarHolder, pairs_to_load: List[pd.DataFrame]): @@ -232,14 +232,14 @@ class LookaheadAnalysis: # check if requirements have been met of full_varholder found_signals: int = self.full_varHolder.result['results'].shape[0] + 1 if found_signals >= self.targeted_trade_amount: - logging.info(f"Found {found_signals} trades, " - f"calculating {self.targeted_trade_amount} trades.") + logger.info(f"Found {found_signals} trades, " + f"calculating {self.targeted_trade_amount} trades.") elif self.targeted_trade_amount >= found_signals >= self.minimum_trade_amount: - logging.info(f"Only found {found_signals} trades. Calculating all available trades.") + logger.info(f"Only found {found_signals} trades. Calculating all available trades.") else: - logging.info(f"found {found_signals} trades " - f"which is less than minimum_trade_amount {self.minimum_trade_amount}. " - f"Cancelling this backtest lookahead bias test.") + logger.info(f"found {found_signals} trades " + f"which is less than minimum_trade_amount {self.minimum_trade_amount}. " + f"Cancelling this backtest lookahead bias test.") return # now we loop through all signals @@ -253,9 +253,9 @@ class LookaheadAnalysis: if (self.current_analysis.false_entry_signals > 0 or self.current_analysis.false_exit_signals > 0 or len(self.current_analysis.false_indicators) > 0): - logging.info(f" => {self.local_config['strategy']} + : bias detected!") + logger.info(f" => {self.local_config['strategy']} + : bias detected!") self.current_analysis.has_bias = True else: - logging.info(self.local_config['strategy'] + ": no bias detected") + logger.info(self.local_config['strategy'] + ": no bias detected") self.failed_bias_check = False From 3e6a2bf9b04d0c140350b0f78baa274dc7bae11d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 May 2023 20:12:04 +0200 Subject: [PATCH 25/55] Add parameters for analysis tests ... --- tests/optimize/test_lookahead_analysis.py | 23 +++++++++++++++++-- .../strategy_test_v3_with_lookahead_bias.py | 17 ++++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index ef4f8809c..0706750ec 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -113,7 +113,10 @@ def test_initialize_single_lookahead_analysis(): pytest.skip("TODO") -def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: +@pytest.mark.parametrize('scenario', [ + 'no_bias', 'bias1' +]) +def test_biased_strategy(lookahead_conf, mocker, caplog, scenario) -> None: mocker.patch('freqtrade.data.history.get_timerange', get_timerange) mocker.patch(f'{EXMS}.get_fee', return_value=0.0) @@ -128,10 +131,26 @@ def test_biased_strategy(lookahead_conf, mocker, caplog) -> None: lookahead_conf['timerange'] = '20180119-20180122' lookahead_conf['strategy'] = 'strategy_test_v3_with_lookahead_bias' + # Patch scenario Parameter to allow for easy selection + mocker.patch('freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file', + return_value={ + 'params': { + "buy": { + "scenario": scenario + } + } + }) + strategy_obj = {} strategy_obj['name'] = "strategy_test_v3_with_lookahead_bias" instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.start() + # Assert init correct + assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog) + # Assert bias detected assert log_has_re(r".*bias detected.*", caplog) - # TODO: assert something ... most likely output (?) or instance state? + + # Assert False to see full logs in output + # assert False + # Run with `pytest tests/optimize/test_lookahead_analysis.py -k test_biased_strategy` diff --git a/tests/strategy/strats/lookahead_bias/strategy_test_v3_with_lookahead_bias.py b/tests/strategy/strats/lookahead_bias/strategy_test_v3_with_lookahead_bias.py index 6cf894586..d35b85b2d 100644 --- a/tests/strategy/strats/lookahead_bias/strategy_test_v3_with_lookahead_bias.py +++ b/tests/strategy/strats/lookahead_bias/strategy_test_v3_with_lookahead_bias.py @@ -3,6 +3,7 @@ from pandas import DataFrame from technical.indicators import ichimoku from freqtrade.strategy import IStrategy +from freqtrade.strategy.parameters import CategoricalParameter class strategy_test_v3_with_lookahead_bias(IStrategy): @@ -21,6 +22,7 @@ class strategy_test_v3_with_lookahead_bias(IStrategy): # Optimal timeframe for the strategy timeframe = '5m' + scenario = CategoricalParameter(['no_bias', 'bias1'], default='bias1', space="buy") # Number of candles the strategy requires before producing valid signals startup_candle_count: int = 20 @@ -37,14 +39,19 @@ class strategy_test_v3_with_lookahead_bias(IStrategy): return dataframe def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe.loc[ - dataframe['close'].shift(-10) > dataframe['close'], - 'enter_long'] = 1 + if self.scenario.value == 'no_bias': + dataframe.loc[dataframe['close'].shift(10) < dataframe['close'], 'enter_long'] = 1 + else: + dataframe.loc[dataframe['close'].shift(-10) > dataframe['close'], 'enter_long'] = 1 return dataframe def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe.loc[ - dataframe['close'].shift(-10) > dataframe['close'], 'exit'] = 1 + if self.scenario.value == 'no_bias': + dataframe.loc[ + dataframe['close'].shift(10) < dataframe['close'], 'exit'] = 1 + else: + dataframe.loc[ + dataframe['close'].shift(-10) > dataframe['close'], 'exit'] = 1 return dataframe From 70a0c2e62527184d8b1af08e09ac57abdde291e2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 08:21:08 +0200 Subject: [PATCH 26/55] Fix test mishap --- tests/optimize/test_lookahead_analysis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 0706750ec..8ee92e6fc 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -20,6 +20,7 @@ def lookahead_conf(default_conf_usdt): default_conf_usdt['targeted_trade_amount'] = 20 default_conf_usdt['strategy_path'] = str( Path(__file__).parent.parent / "strategy/strats/lookahead_bias") + default_conf_usdt['strategy'] = 'strategy_test_v3_with_lookahead_bias' return default_conf_usdt @@ -37,7 +38,7 @@ def test_start_lookahead_analysis(mocker): "--strategy", "strategy_test_v3_with_lookahead_bias", "--strategy-path", - str(Path(__file__).parent.parent / "strategy" / "strats"), + str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), ] pargs = get_args(args) pargs['config'] = None @@ -54,7 +55,7 @@ def test_start_lookahead_analysis(mocker): "--strategy", "strategy_test_v3_with_lookahead_bias", "--strategy-path", - str(Path(__file__).parent.parent / "strategy" / "strats"), + str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), "--targeted-trade-amount", "10", "--minimum-trade-amount", @@ -129,7 +130,6 @@ def test_biased_strategy(lookahead_conf, mocker, caplog, scenario) -> None: lookahead_conf['timeframe'] = '5m' lookahead_conf['timerange'] = '20180119-20180122' - lookahead_conf['strategy'] = 'strategy_test_v3_with_lookahead_bias' # Patch scenario Parameter to allow for easy selection mocker.patch('freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file', From eb31b574c1f362b7a9914275a8e19ba0f0cf9c80 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Fri, 26 May 2023 12:55:54 +0200 Subject: [PATCH 27/55] added returns to text_table_lookahead_analysis_instances filled in test_lookahead_helper_text_table_lookahead_analysis_instances --- .../optimize/lookahead_analysis_helpers.py | 1 + tests/optimize/test_lookahead_analysis.py | 68 +++++++++++++++---- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 53d8c8a59..54c63b78c 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -44,6 +44,7 @@ class LookaheadAnalysisSubFunctions: from tabulate import tabulate table = tabulate(data, headers=headers, tablefmt="orgtbl") print(table) + return table, headers, data @staticmethod def export_to_csv(config: Dict[str, Any], lookahead_analysis: List[LookaheadAnalysis]): diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 8ee92e6fc..e9c5f0f85 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument from copy import deepcopy -from pathlib import Path +from pathlib import Path, PurePosixPath from unittest.mock import MagicMock, PropertyMock import pytest @@ -9,7 +9,7 @@ import pytest from freqtrade.commands.optimize_commands import start_lookahead_analysis from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException -from freqtrade.optimize.lookahead_analysis import LookaheadAnalysis +from freqtrade.optimize.lookahead_analysis import Analysis, LookaheadAnalysis from freqtrade.optimize.lookahead_analysis_helpers import LookaheadAnalysisSubFunctions from tests.conftest import EXMS, get_args, log_has_re, patch_exchange @@ -32,7 +32,7 @@ def test_start_lookahead_analysis(mocker): 'freqtrade.optimize.lookahead_analysis_helpers.LookaheadAnalysisSubFunctions', initialize_single_lookahead_analysis=single_mock, text_table_lookahead_analysis_instances=text_table_mock, - ) + ) args = [ "lookahead-analysis", "--strategy", @@ -60,7 +60,7 @@ def test_start_lookahead_analysis(mocker): "10", "--minimum-trade-amount", "20", - ] + ] pargs = get_args(args) pargs['config'] = None with pytest.raises(OperationalException, @@ -87,10 +87,10 @@ def test_lookahead_helper_start(lookahead_conf, mocker, caplog) -> None: single_mock = MagicMock() text_table_mock = MagicMock() mocker.patch.multiple( - 'freqtrade.optimize.lookahead_analysis_helpers.LookaheadAnalysisSubFunctions', - initialize_single_lookahead_analysis=single_mock, - text_table_lookahead_analysis_instances=text_table_mock, - ) + 'freqtrade.optimize.lookahead_analysis_helpers.LookaheadAnalysisSubFunctions', + initialize_single_lookahead_analysis=single_mock, + text_table_lookahead_analysis_instances=text_table_mock, + ) LookaheadAnalysisSubFunctions.start(lookahead_conf) assert single_mock.call_count == 1 assert text_table_mock.call_count == 1 @@ -99,9 +99,52 @@ def test_lookahead_helper_start(lookahead_conf, mocker, caplog) -> None: text_table_mock.reset_mock() -def test_lookahead_helper_text_table_lookahead_analysis_instances(): - # TODO - pytest.skip("TODO") +def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf, caplog): + analysis = Analysis() + analysis.total_signals = 5 + analysis.has_bias = True + analysis.false_entry_signals = 4 + analysis.false_exit_signals = 3 + + strategy_obj = \ + { + 'name': "strategy_test_v3_with_lookahead_bias", + 'location': PurePosixPath(lookahead_conf['strategy_path'], + f"{lookahead_conf['strategy']}.py") + } + + instance = LookaheadAnalysis(lookahead_conf, strategy_obj) + instance.current_analysis = analysis + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances([instance])) + + # check amount of returning rows + assert len(data) == 1 + + # check row contents for a try that errored out + assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' + assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' + assert data[0][2].__contains__('error') + assert len(data[0]) == 3 + + # edit it into not showing an error + instance.failed_bias_check = False + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances([instance])) + assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' + assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' + assert data[0][2] # True + assert data[0][3] == 5 + assert data[0][4] == 4 + assert data[0][5] == 3 + assert data[0][6] == '' + + analysis.false_indicators.append('falseIndicator1') + analysis.false_indicators.append('falseIndicator2') + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances([instance])) + + assert data[0][6] == 'falseIndicator1, falseIndicator2' def test_lookahead_helper_export_to_csv(): @@ -118,7 +161,6 @@ def test_initialize_single_lookahead_analysis(): 'no_bias', 'bias1' ]) def test_biased_strategy(lookahead_conf, mocker, caplog, scenario) -> None: - mocker.patch('freqtrade.data.history.get_timerange', get_timerange) mocker.patch(f'{EXMS}.get_fee', return_value=0.0) mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) @@ -136,7 +178,7 @@ def test_biased_strategy(lookahead_conf, mocker, caplog, scenario) -> None: return_value={ 'params': { "buy": { - "scenario": scenario + "scenario": scenario } } }) From 636298bb719e0fde0395a05ee5f8f9790c1df977 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sat, 27 May 2023 19:15:35 +0200 Subject: [PATCH 28/55] added test_lookahead_helper_export_to_csv --- tests/optimize/test_lookahead_analysis.py | 134 ++++++++++++++++++++-- 1 file changed, 126 insertions(+), 8 deletions(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index e9c5f0f85..85cd8fd66 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument - from copy import deepcopy from pathlib import Path, PurePosixPath from unittest.mock import MagicMock, PropertyMock @@ -101,8 +100,8 @@ def test_lookahead_helper_start(lookahead_conf, mocker, caplog) -> None: def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf, caplog): analysis = Analysis() - analysis.total_signals = 5 analysis.has_bias = True + analysis.total_signals = 5 analysis.false_entry_signals = 4 analysis.false_exit_signals = 3 @@ -118,9 +117,6 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf table, headers, data = (LookaheadAnalysisSubFunctions. text_table_lookahead_analysis_instances([instance])) - # check amount of returning rows - assert len(data) == 1 - # check row contents for a try that errored out assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' @@ -146,10 +142,132 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf assert data[0][6] == 'falseIndicator1, falseIndicator2' + # check amount of returning rows + assert len(data) == 1 -def test_lookahead_helper_export_to_csv(): - # TODO - pytest.skip("TODO") + # check amount of multiple rows + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances([instance, instance, instance])) + assert len(data) == 3 + + +def test_lookahead_helper_export_to_csv(lookahead_conf): + import pandas as pd + lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" + + # just to be sure the test won't fail: remove file if exists for some reason + # (repeat this at the end once again to clean up) + if Path(lookahead_conf['lookahead_analysis_exportfilename']).exists(): + Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() + + # before we can start we have to delete the + + # 1st check: create a new file and verify its contents + analysis1 = Analysis() + analysis1.has_bias = True + analysis1.total_signals = 5 + analysis1.false_entry_signals = 4 + analysis1.false_exit_signals = 3 + analysis1.false_indicators.append('falseIndicator1') + analysis1.false_indicators.append('falseIndicator2') + lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" + + strategy_obj1 = { + 'name': "strat1", + 'location': PurePosixPath("file1.py"), + } + + instance1 = LookaheadAnalysis(lookahead_conf, strategy_obj1) + instance1.current_analysis = analysis1 + + LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance1]) + saved_data1 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) + + expected_values1 = [ + [ + 'file1.py', 'strat1', True, + 5, 4, 3, + "falseIndicator1,falseIndicator2" + ], + ] + expected_columns = ['filename', 'strategy', 'has_bias', + 'total_signals', 'biased_entry_signals', 'biased_exit_signals', + 'biased_indicators'] + expected_data1 = pd.DataFrame(expected_values1, columns=expected_columns) + + assert Path(lookahead_conf['lookahead_analysis_exportfilename']).exists() + assert expected_data1.equals(saved_data1) + + # 2nd check: update the same strategy (which internally changed or is being retested) + expected_values2 = [ + [ + 'file1.py', 'strat1', False, + 10, 11, 12, + "falseIndicator3,falseIndicator4" + ], + ] + expected_data2 = pd.DataFrame(expected_values2, columns=expected_columns) + + analysis2 = Analysis() + analysis2.has_bias = False + analysis2.total_signals = 10 + analysis2.false_entry_signals = 11 + analysis2.false_exit_signals = 12 + analysis2.false_indicators.append('falseIndicator3') + analysis2.false_indicators.append('falseIndicator4') + + strategy_obj2 = { + 'name': "strat1", + 'location': PurePosixPath("file1.py"), + } + + instance2 = LookaheadAnalysis(lookahead_conf, strategy_obj2) + instance2.current_analysis = analysis2 + + LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance2]) + saved_data2 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) + + assert expected_data2.equals(saved_data2) + + # 3rd check: now we add a new row to an already existing file + expected_values3 = [ + [ + 'file1.py', 'strat1', False, + 10, 11, 12, + "falseIndicator3,falseIndicator4" + ], + [ + 'file3.py', 'strat3', True, + 20, 21, 22, "falseIndicator5,falseIndicator6" + ], + ] + + expected_data3 = pd.DataFrame(expected_values3, columns=expected_columns) + + analysis3 = Analysis() + analysis3.has_bias = True + analysis3.total_signals = 20 + analysis3.false_entry_signals = 21 + analysis3.false_exit_signals = 22 + analysis3.false_indicators.append('falseIndicator5') + analysis3.false_indicators.append('falseIndicator6') + lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" + + strategy_obj3 = { + 'name': "strat3", + 'location': PurePosixPath("file3.py"), + } + + instance3 = LookaheadAnalysis(lookahead_conf, strategy_obj3) + instance3.current_analysis = analysis3 + + LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance3]) + saved_data3 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) + assert expected_data3.equals(saved_data3) + + # remove csv file after the test is done + if Path(lookahead_conf['lookahead_analysis_exportfilename']).exists(): + Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() def test_initialize_single_lookahead_analysis(): From a7426755bc5892c0d9a4dc98a7f536612a6ec296 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sat, 27 May 2023 20:35:45 +0200 Subject: [PATCH 29/55] added a check for bias1. Looking at has_bias should be enough to statisfy the test. The tests could be extended with thecking the buy/sell signals and the dataframe itself - but this should be sufficient for now. --- tests/optimize/test_lookahead_analysis.py | 39 +++++++++++++------ .../strategy_test_v3_with_lookahead_bias.py | 13 ++++--- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 85cd8fd66..1bd864906 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -173,9 +173,9 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" strategy_obj1 = { - 'name': "strat1", - 'location': PurePosixPath("file1.py"), - } + 'name': "strat1", + 'location': PurePosixPath("file1.py"), + } instance1 = LookaheadAnalysis(lookahead_conf, strategy_obj1) instance1.current_analysis = analysis1 @@ -270,9 +270,24 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() -def test_initialize_single_lookahead_analysis(): - # TODO - pytest.skip("TODO") +def test_initialize_single_lookahead_analysis(lookahead_conf, mocker): + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + mocker.patch(f'{EXMS}.get_fee', return_value=0.0) + mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) + mocker.patch(f'{EXMS}.get_max_pair_stake_amount', return_value=float('inf')) + patch_exchange(mocker) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + lookahead_conf['pairs'] = ['UNITTEST/USDT'] + + lookahead_conf['timeframe'] = '5m' + lookahead_conf['timerange'] = '20180119-20180122' + strategy_obj = { + 'name': "strat1", + 'location': PurePosixPath("file1.py"), + } + LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( + strategy_obj, lookahead_conf) @pytest.mark.parametrize('scenario', [ @@ -307,10 +322,10 @@ def test_biased_strategy(lookahead_conf, mocker, caplog, scenario) -> None: instance.start() # Assert init correct assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog) - # Assert bias detected - assert log_has_re(r".*bias detected.*", caplog) - # TODO: assert something ... most likely output (?) or instance state? - # Assert False to see full logs in output - # assert False - # Run with `pytest tests/optimize/test_lookahead_analysis.py -k test_biased_strategy` + # check non-biased strategy + if scenario == "no_bias": + assert not instance.current_analysis.has_bias + # check biased strategy + elif scenario == "bias1": + assert instance.current_analysis.has_bias diff --git a/tests/strategy/strats/lookahead_bias/strategy_test_v3_with_lookahead_bias.py b/tests/strategy/strats/lookahead_bias/strategy_test_v3_with_lookahead_bias.py index d35b85b2d..e50d5d17b 100644 --- a/tests/strategy/strats/lookahead_bias/strategy_test_v3_with_lookahead_bias.py +++ b/tests/strategy/strats/lookahead_bias/strategy_test_v3_with_lookahead_bias.py @@ -29,12 +29,13 @@ class strategy_test_v3_with_lookahead_bias(IStrategy): def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # bias is introduced here - ichi = ichimoku(dataframe, - conversion_line_period=20, - base_line_periods=60, - laggin_span=120, - displacement=30) - dataframe['chikou_span'] = ichi['chikou_span'] + if self.scenario.value != 'no_bias': + ichi = ichimoku(dataframe, + conversion_line_period=20, + base_line_periods=60, + laggin_span=120, + displacement=30) + dataframe['chikou_span'] = ichi['chikou_span'] return dataframe From 0ed84fbcc18bf6e5ecbb6667831348f94c95d32d Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sat, 27 May 2023 20:47:59 +0200 Subject: [PATCH 30/55] added test_initialize_single_lookahead_analysis A check for a random variable should be enough, right? :) --- tests/optimize/test_lookahead_analysis.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 1bd864906..814c9f3b8 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -282,12 +282,10 @@ def test_initialize_single_lookahead_analysis(lookahead_conf, mocker): lookahead_conf['timeframe'] = '5m' lookahead_conf['timerange'] = '20180119-20180122' - strategy_obj = { - 'name': "strat1", - 'location': PurePosixPath("file1.py"), - } - LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( - strategy_obj, lookahead_conf) + strategy_obj = {'name': "strategy_test_v3_with_lookahead_bias"} + + instance = LookaheadAnalysis(lookahead_conf, strategy_obj) + assert instance.strategy_obj['name'] == "strategy_test_v3_with_lookahead_bias" @pytest.mark.parametrize('scenario', [ @@ -316,8 +314,7 @@ def test_biased_strategy(lookahead_conf, mocker, caplog, scenario) -> None: } }) - strategy_obj = {} - strategy_obj['name'] = "strategy_test_v3_with_lookahead_bias" + strategy_obj = {'name': "strategy_test_v3_with_lookahead_bias"} instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.start() # Assert init correct From 9bb25be88091c3455c7a8e60b1edca5fe1d4eab1 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sat, 27 May 2023 22:31:47 +0200 Subject: [PATCH 31/55] modified help-string for the cli-option lookahead_analysis_exportfilename moved doc from utils.md to lookahead-analysis.md and modified it (unfinished) added a check to automatically edit the config['backtest_cache'] to be 'none' --- docs/lookahead-analysis.md | 79 +++++++++++++++++++ docs/utils.md | 33 -------- freqtrade/commands/cli_options.py | 2 +- .../optimize/lookahead_analysis_helpers.py | 11 +++ 4 files changed, 91 insertions(+), 34 deletions(-) create mode 100644 docs/lookahead-analysis.md diff --git a/docs/lookahead-analysis.md b/docs/lookahead-analysis.md new file mode 100644 index 000000000..22440a6d6 --- /dev/null +++ b/docs/lookahead-analysis.md @@ -0,0 +1,79 @@ +# Lookahead analysis +This page explains how to validate your strategy in terms of look ahead bias. + +Checking look ahead bias is the bane of any strategy since it is sometimes very easy to introduce backtest bias - +but very hard to detect. + +Backtesting initializes all timestamps at once and calculates all indicators in the beginning. +This means that if you are allowing your indicators (or the libraries that get used) then you would +look into the future and falsify your backtest. + +Lookahead-analysis requires historic data to be available. +To learn how to get data for the pairs and exchange you're interested in, +head over to the [Data Downloading](data-download.md) section of the documentation. + +This command is built upon backtesting +since it internally chains backtests and pokes at the strategy to provoke it to show look ahead bias. +This is done by looking not at the strategy itself - but at the results it returned. +The results are things like changed indicator-values and moved entries/exits compared to the full backtest. + +You can use commands of [Backtesting](backtesting.md). +It also supports the lookahead-analysis of freqai strategies. + +--cache is enforced to be "none" + +## Backtesting command reference + +``` +usage: freqtrade lookahead-analysis [-h] [-v] [-V] + [--minimum-trade-amount INT] + [--targeted-trade-amount INT] + [--lookahead-analysis-exportfilename PATH] + +optional arguments: + -h, --help show this help message and exit + --minimum-trade-amount INT + Override the value of the `minimum_trade_amount` configuration + setting + Requires `--targeted-trade-amount` to be larger or equal to --minimum-trade-amount. + (default: 10) + --targeted-trade-amount INT + Override the value of the `minimum_trade_amount` configuration + (default: 20) + --lookahead-analysis-exportfilename PATH + Use this filename to save your lookahead-analysis-results to a csv file +``` + + +#### Summary +Checks a given strategy for look ahead bias via backtest-analysis +Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting +and producing false hopes for the one backtesting. + +#### Introduction: +Many strategies - without the programmer knowing - have fallen prey to look ahead bias. + +Any backtest will populate the full dataframe including all time stamps at the beginning. +If the programmer is not careful or oblivious how things work internally +(which sometimes can be really hard to find out) then it will just look into the future making the strategy amazing +but not realistic. + +This command is made to try to verify the validity in the form of the aforementioned look ahead bias. + +#### How does the command work? +It will not look at the strategy or any contents itself but instead will run multiple backtests +by using precisely cut timeranges and analyzing the results each time, comparing to the full timerange. + +At first, it starts a backtest over the whole duration +and then repeats backtests from the same starting point to the respective points to watch. +In addition, it analyzes the dataframes form the overall backtest to the cut ones. + +At the end it will return a result-table in terminal. + +Hint: +If an entry or exit condition is only triggered rarely or the timerange was chosen +so only a few entry conditions are met +then the bias checker is unable to catch the biased entry or exit condition. +In the end it only checks which entry and exit signals have been triggered. + +---Flow chart here for better understanding--- diff --git a/docs/utils.md b/docs/utils.md index 798a87fae..900856af4 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -1010,36 +1010,3 @@ Common arguments: Path to userdata directory. ``` -### Lookahead - analysis -#### Summary -Checks a given strategy for look ahead bias via backtest-analysis -Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting -and producing false hopes for the one backtesting. - -#### Introduction: -Many strategies - without the programmer knowing - have fallen prey to look ahead bias. - -Any backtest will populate the full dataframe including all time stamps at the beginning. -If the programmer is not careful or oblivious how things work internally -(which sometimes can be really hard to find out) then it will just look into the future making the strategy amazing -but not realistic. - -The tool is made to try to verify the validity in the form of the aforementioned look ahead bias. - -#### How does the command work? -It will not look at the strategy or any contents itself but instead will run multiple backtests -by using precisely cut timeranges and analyzing the results each time, comparing to the full timerange. - -At first, it starts a backtest over the whole duration -and then repeats backtests from the same starting point to the respective points to watch. -In addition, it analyzes the dataframes form the overall backtest to the cut ones. - -At the end it will return a result-table in terminal. - -Hint: -If an entry or exit condition is only triggered rarely or the timerange was chosen -so only a few entry conditions are met -then the bias checker is unable to catch the biased entry or exit condition. -In the end it only checks which entry and exit signals have been triggered. - ----Flow chart here for better understanding--- diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index e4a864ea0..08283430e 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -704,7 +704,7 @@ AVAILABLE_CLI_OPTIONS = { ), "lookahead_analysis_exportfilename": Arg( '--lookahead-analysis-exportfilename', - help="Use this filename to store lookahead-analysis-results", + help="Use this csv-filename to store lookahead-analysis-results", type=str ), } diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 54c63b78c..f212d8403 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -110,6 +110,17 @@ class LookaheadAnalysisSubFunctions: "targeted trade amount can't be smaller than minimum trade amount." ) + # enforce cache to be 'none', shift it to 'none' if not already + # (since the default value is 'day') + if config.get('backtest_cache') is None: + config['backtest_cache'] = 'none' + elif config['backtest_cache'] != 'none': + logger.info(f"backtest_cache = " + f"{config['backtest_cache']} detected. " + f"Inside lookahead-analysis it is enforced to be 'none'. " + f"Changed it to 'none'") + config['backtest_cache'] = 'none' + strategy_objs = StrategyResolver.search_all_objects( config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) From eec78371672e11886662a10daf8cf5e5a8363abe Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 28 May 2023 20:52:58 +0200 Subject: [PATCH 32/55] - modified help-string for the cli-option lookahead_analysis_exportfilename - moved doc from utils.md to lookahead-analysis.md and modified it (unfinished) - added a check to automatically edit the config['backtest_cache'] to be 'none' - adjusted test_lookahead_helper_export_to_csv to catch the new catching of errors - adjusted test_lookahead_helper_text_table_lookahead_analysis_instances to catch the new catching of errors - changed lookahead_analysis.start result-reporting to show that not enough trades were caught including x of y --- freqtrade/optimize/lookahead_analysis.py | 19 ++++-- .../optimize/lookahead_analysis_helpers.py | 43 ++++++++++---- tests/optimize/test_lookahead_analysis.py | 59 +++++++++++-------- 3 files changed, 80 insertions(+), 41 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index e40322c88..4f3d7a4d0 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -250,12 +250,19 @@ class LookaheadAnalysis: self.analyze_row(idx, result_row) # check and report signals - if (self.current_analysis.false_entry_signals > 0 or - self.current_analysis.false_exit_signals > 0 or - len(self.current_analysis.false_indicators) > 0): - logger.info(f" => {self.local_config['strategy']} + : bias detected!") + if self.current_analysis.total_signals < self.local_config['minimum_trade_amount']: + logger.info(f" -> {self.local_config['strategy']} : too few trades. " + f"We only found {self.current_analysis.total_signals} trades. " + f"Hint: Extend the timerange " + f"to get at least {self.local_config['minimum_trade_amount']} " + f"or lower the value of minimum_trade_amount.") + self.failed_bias_check = True + elif (self.current_analysis.false_entry_signals > 0 or + self.current_analysis.false_exit_signals > 0 or + len(self.current_analysis.false_indicators) > 0): + logger.info(f" => {self.local_config['strategy']} : bias detected!") self.current_analysis.has_bias = True + self.failed_bias_check = False else: logger.info(self.local_config['strategy'] + ": no bias detected") - - self.failed_bias_check = False + self.failed_bias_check = False diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index f212d8403..49f225943 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -16,12 +16,24 @@ logger = logging.getLogger(__name__) class LookaheadAnalysisSubFunctions: @staticmethod - def text_table_lookahead_analysis_instances(lookahead_instances: List[LookaheadAnalysis]): + def text_table_lookahead_analysis_instances( + config: Dict[str, Any], + lookahead_instances: List[LookaheadAnalysis]): headers = ['filename', 'strategy', 'has_bias', 'total_signals', 'biased_entry_signals', 'biased_exit_signals', 'biased_indicators'] data = [] for inst in lookahead_instances: - if inst.failed_bias_check: + if config['minimum_trade_amount'] > inst.current_analysis.total_signals: + data.append( + [ + inst.strategy_obj['location'].parts[-1], + inst.strategy_obj['name'], + "too few trades caught " + f"({inst.current_analysis.total_signals}/{config['minimum_trade_amount']})." + f"Test failed." + ] + ) + elif inst.failed_bias_check: data.append( [ inst.strategy_obj['location'].parts[-1], @@ -77,14 +89,21 @@ class LookaheadAnalysisSubFunctions: index=None) for inst in lookahead_analysis: - new_row_data = {'filename': inst.strategy_obj['location'].parts[-1], - 'strategy': inst.strategy_obj['name'], - 'has_bias': inst.current_analysis.has_bias, - 'total_signals': inst.current_analysis.total_signals, - 'biased_entry_signals': inst.current_analysis.false_entry_signals, - 'biased_exit_signals': inst.current_analysis.false_exit_signals, - 'biased_indicators': ",".join(inst.current_analysis.false_indicators)} - csv_df = add_or_update_row(csv_df, new_row_data) + # only update if + if (inst.current_analysis.total_signals > config['minimum_trade_amount'] + and inst.failed_bias_check is not True): + new_row_data = {'filename': inst.strategy_obj['location'].parts[-1], + 'strategy': inst.strategy_obj['name'], + 'has_bias': inst.current_analysis.has_bias, + 'total_signals': + int(inst.current_analysis.total_signals), + 'biased_entry_signals': + int(inst.current_analysis.false_entry_signals), + 'biased_exit_signals': + int(inst.current_analysis.false_exit_signals), + 'biased_indicators': + ",".join(inst.current_analysis.false_indicators)} + csv_df = add_or_update_row(csv_df, new_row_data) logger.info(f"saving {config['lookahead_analysis_exportfilename']}") csv_df.to_csv(config['lookahead_analysis_exportfilename'], index=False) @@ -122,7 +141,7 @@ class LookaheadAnalysisSubFunctions: config['backtest_cache'] = 'none' strategy_objs = StrategyResolver.search_all_objects( - config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) + config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) lookaheadAnalysis_instances = [] @@ -147,7 +166,7 @@ class LookaheadAnalysisSubFunctions: # report the results if lookaheadAnalysis_instances: LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( - lookaheadAnalysis_instances) + config, lookaheadAnalysis_instances) if config.get('lookahead_analysis_exportfilename') is not None: LookaheadAnalysisSubFunctions.export_to_csv(config, lookaheadAnalysis_instances) else: diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 814c9f3b8..476627c57 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -115,30 +115,40 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.current_analysis = analysis table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances([instance])) + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) - # check row contents for a try that errored out + # check row contents for a try that has too few signals assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' - assert data[0][2].__contains__('error') + assert data[0][2].__contains__('too few trades') assert len(data[0]) == 3 + # now check for an error which occured after enough trades + analysis.total_signals = 12 + analysis.false_entry_signals = 11 + analysis.false_exit_signals = 10 + instance = LookaheadAnalysis(lookahead_conf, strategy_obj) + instance.current_analysis = analysis + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + assert data[0][2].__contains__("error") + # edit it into not showing an error instance.failed_bias_check = False table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances([instance])) + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' assert data[0][2] # True - assert data[0][3] == 5 - assert data[0][4] == 4 - assert data[0][5] == 3 + assert data[0][3] == 12 + assert data[0][4] == 11 + assert data[0][5] == 10 assert data[0][6] == '' analysis.false_indicators.append('falseIndicator1') analysis.false_indicators.append('falseIndicator2') table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances([instance])) + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) assert data[0][6] == 'falseIndicator1, falseIndicator2' @@ -146,8 +156,8 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf assert len(data) == 1 # check amount of multiple rows - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances([instance, instance, instance])) + table, headers, data = (LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + lookahead_conf, [instance, instance, instance])) assert len(data) == 3 @@ -165,9 +175,9 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): # 1st check: create a new file and verify its contents analysis1 = Analysis() analysis1.has_bias = True - analysis1.total_signals = 5 - analysis1.false_entry_signals = 4 - analysis1.false_exit_signals = 3 + analysis1.total_signals = 12 + analysis1.false_entry_signals = 11 + analysis1.false_exit_signals = 10 analysis1.false_indicators.append('falseIndicator1') analysis1.false_indicators.append('falseIndicator2') lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" @@ -178,6 +188,7 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): } instance1 = LookaheadAnalysis(lookahead_conf, strategy_obj1) + instance1.failed_bias_check = False instance1.current_analysis = analysis1 LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance1]) @@ -186,7 +197,7 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): expected_values1 = [ [ 'file1.py', 'strat1', True, - 5, 4, 3, + 12, 11, 10, "falseIndicator1,falseIndicator2" ], ] @@ -202,7 +213,7 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): expected_values2 = [ [ 'file1.py', 'strat1', False, - 10, 11, 12, + 22, 21, 20, "falseIndicator3,falseIndicator4" ], ] @@ -210,9 +221,9 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): analysis2 = Analysis() analysis2.has_bias = False - analysis2.total_signals = 10 - analysis2.false_entry_signals = 11 - analysis2.false_exit_signals = 12 + analysis2.total_signals = 22 + analysis2.false_entry_signals = 21 + analysis2.false_exit_signals = 20 analysis2.false_indicators.append('falseIndicator3') analysis2.false_indicators.append('falseIndicator4') @@ -222,6 +233,7 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): } instance2 = LookaheadAnalysis(lookahead_conf, strategy_obj2) + instance2.failed_bias_check = False instance2.current_analysis = analysis2 LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance2]) @@ -233,12 +245,12 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): expected_values3 = [ [ 'file1.py', 'strat1', False, - 10, 11, 12, + 22, 21, 20, "falseIndicator3,falseIndicator4" ], [ 'file3.py', 'strat3', True, - 20, 21, 22, "falseIndicator5,falseIndicator6" + 32, 31, 30, "falseIndicator5,falseIndicator6" ], ] @@ -246,9 +258,9 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): analysis3 = Analysis() analysis3.has_bias = True - analysis3.total_signals = 20 - analysis3.false_entry_signals = 21 - analysis3.false_exit_signals = 22 + analysis3.total_signals = 32 + analysis3.false_entry_signals = 31 + analysis3.false_exit_signals = 30 analysis3.false_indicators.append('falseIndicator5') analysis3.false_indicators.append('falseIndicator6') lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" @@ -259,6 +271,7 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): } instance3 = LookaheadAnalysis(lookahead_conf, strategy_obj3) + instance3.failed_bias_check = False instance3.current_analysis = analysis3 LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance3]) From 6b3b5f201d7e3b00bfc54d885aa89dadba202103 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 28 May 2023 22:13:29 +0200 Subject: [PATCH 33/55] export_to_csv: Added forced conversion of float64 to int to remove the .0 values once and for all ... --- freqtrade/optimize/lookahead_analysis_helpers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 49f225943..0eccf0526 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -105,6 +105,16 @@ class LookaheadAnalysisSubFunctions: ",".join(inst.current_analysis.false_indicators)} csv_df = add_or_update_row(csv_df, new_row_data) + # Fill NaN values with a default value (e.g., 0) + csv_df['total_signals'] = csv_df['total_signals'].fillna(0) + csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].fillna(0) + csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].fillna(0) + + # Convert columns to integers + csv_df['total_signals'] = csv_df['total_signals'].astype(int) + csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].astype(int) + csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].astype(int) + logger.info(f"saving {config['lookahead_analysis_exportfilename']}") csv_df.to_csv(config['lookahead_analysis_exportfilename'], index=False) From 6b736c49d4366618ff5d2cc4accc983c44ef4c66 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Jun 2023 20:13:28 +0200 Subject: [PATCH 34/55] Dont persist Backtesting to avoid memory leak --- freqtrade/optimize/lookahead_analysis.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 4f3d7a4d0..a567b3b83 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -61,7 +61,7 @@ class LookaheadAnalysis: return timestamp @staticmethod - def get_result(backtesting, processed: pd.DataFrame): + def get_result(backtesting: Backtesting, processed: pd.DataFrame): min_date, max_date = get_timerange(processed) result = backtesting.backtest( @@ -143,15 +143,15 @@ class LookaheadAnalysis: str(self.dt_to_timestamp(varholder.to_dt))) prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load - self.backtesting = Backtesting(prepare_data_config) - self.backtesting._set_strategy(self.backtesting.strategylist[0]) + backtesting = Backtesting(prepare_data_config) + backtesting._set_strategy(backtesting.strategylist[0]) - varholder.data, varholder.timerange = self.backtesting.load_bt_data() - self.backtesting.load_bt_data_detail() - varholder.timeframe = self.backtesting.timeframe + varholder.data, varholder.timerange = backtesting.load_bt_data() + backtesting.load_bt_data_detail() + varholder.timeframe = backtesting.timeframe - varholder.indicators = self.backtesting.strategy.advise_all_indicators(varholder.data) - varholder.result = self.get_result(self.backtesting, varholder.indicators) + varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) + varholder.result = self.get_result(backtesting, varholder.indicators) def fill_full_varholder(self): self.full_varHolder = VarHolder() From 05ea36f03b02ff896cd84c7e876579710082c710 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Jun 2023 06:45:34 +0200 Subject: [PATCH 35/55] Fix performance when running tons of backtests --- freqtrade/optimize/backtesting.py | 31 +++++++++++++++--------- freqtrade/optimize/lookahead_analysis.py | 4 ++- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d77fc469b..4a5536e84 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -24,6 +24,7 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import (amount_to_contract_precision, price_to_precision, timeframe_to_minutes, timeframe_to_seconds) +from freqtrade.exchange.exchange import Exchange from freqtrade.mixins import LoggingMixin from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.bt_progress import BTProgress @@ -72,7 +73,7 @@ class Backtesting: backtesting.start() """ - def __init__(self, config: Config) -> None: + def __init__(self, config: Config, exchange: Optional[Exchange] = None) -> None: LoggingMixin.show_output = False self.config = config @@ -89,7 +90,10 @@ class Backtesting: self.rejected_df: Dict[str, Dict] = {} self._exchange_name = self.config['exchange']['name'] - self.exchange = ExchangeResolver.load_exchange(self.config, load_leverage_tiers=True) + if not exchange: + exchange = ExchangeResolver.load_exchange(self.config, load_leverage_tiers=True) + self.exchange = exchange + self.dataprovider = DataProvider(self.config, self.exchange) if self.config.get('strategy_list'): @@ -114,16 +118,7 @@ class Backtesting: self.timeframe_min = timeframe_to_minutes(self.timeframe) self.init_backtest_detail() self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider) - if 'VolumePairList' in self.pairlists.name_list: - raise OperationalException("VolumePairList not allowed for backtesting. " - "Please use StaticPairList instead.") - if 'PerformanceFilter' in self.pairlists.name_list: - raise OperationalException("PerformanceFilter not allowed for backtesting.") - - if len(self.strategylist) > 1 and 'PrecisionFilter' in self.pairlists.name_list: - raise OperationalException( - "PrecisionFilter not allowed for backtesting multiple strategies." - ) + self._validate_pairlists_for_backtesting() self.dataprovider.add_pairlisthandler(self.pairlists) self.pairlists.refresh_pairlist() @@ -164,6 +159,18 @@ class Backtesting: self.init_backtest() + def _validate_pairlists_for_backtesting(self): + if 'VolumePairList' in self.pairlists.name_list: + raise OperationalException("VolumePairList not allowed for backtesting. " + "Please use StaticPairList instead.") + if 'PerformanceFilter' in self.pairlists.name_list: + raise OperationalException("PerformanceFilter not allowed for backtesting.") + + if len(self.strategylist) > 1 and 'PrecisionFilter' in self.pairlists.name_list: + raise OperationalException( + "PrecisionFilter not allowed for backtesting multiple strategies." + ) + @staticmethod def cleanup(): LoggingMixin.show_output = True diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index a567b3b83..ca419f7e6 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -46,6 +46,7 @@ class LookaheadAnalysis: self.entry_varHolders: List[VarHolder] = [] self.exit_varHolders: List[VarHolder] = [] + self.exchange = None # pull variables the scope of the lookahead_analysis-instance self.local_config = deepcopy(config) @@ -143,7 +144,8 @@ class LookaheadAnalysis: str(self.dt_to_timestamp(varholder.to_dt))) prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load - backtesting = Backtesting(prepare_data_config) + backtesting = Backtesting(prepare_data_config, self.exchange) + self.exchange = backtesting.exchange backtesting._set_strategy(backtesting.strategylist[0]) varholder.data, varholder.timerange = backtesting.load_bt_data() From b89390c06bc42b210c63fbfb5662539ecbae8a4e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Jun 2023 07:13:45 +0200 Subject: [PATCH 36/55] Reduce log verbosity during bias tester runs --- freqtrade/commands/optimize_commands.py | 2 -- freqtrade/loggers/set_log_levels.py | 29 ++++++++++++++++++++++++ freqtrade/optimize/lookahead_analysis.py | 6 +++++ tests/test_log_setup.py | 18 +++++++++++++++ 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 4b8763737..cdddf0fe5 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -144,5 +144,3 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) LookaheadAnalysisSubFunctions.start(config) - - diff --git a/freqtrade/loggers/set_log_levels.py b/freqtrade/loggers/set_log_levels.py index acd8df379..da046f439 100644 --- a/freqtrade/loggers/set_log_levels.py +++ b/freqtrade/loggers/set_log_levels.py @@ -2,6 +2,9 @@ import logging +logger = logging.getLogger(__name__) + + def set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None: """ Set the logging level for third party libraries @@ -23,3 +26,29 @@ def set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None: logging.getLogger('werkzeug').setLevel( logging.ERROR if api_verbosity == 'error' else logging.INFO ) + + +__BIAS_TESTER_LOGGERS = [ + 'freqtrade.resolvers', + 'freqtrade.strategy.hyper', +] + + +def reduce_verbosity_for_bias_tester() -> None: + """ + Reduce verbosity for bias tester. + It loads the same strategy several times, which would spam the log. + """ + logger.info("Reducing verbosity for bias tester.") + for logger_name in __BIAS_TESTER_LOGGERS: + logging.getLogger(logger_name).setLevel(logging.WARNING) + + +def restore_verbosity_for_bias_tester() -> None: + """ + Restore verbosity after bias tester. + """ + logger.info("Restoring log verbosity.") + log_level = logging.getLogger('freqtrade').getEffectiveLevel() + for logger_name in __BIAS_TESTER_LOGGERS: + logging.getLogger(logger_name).setLevel(log_level) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index ca419f7e6..e98eebeef 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -11,6 +11,8 @@ import pandas as pd from freqtrade.configuration import TimeRange from freqtrade.data.history import get_timerange from freqtrade.exchange import timeframe_to_minutes +from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, + restore_verbosity_for_bias_tester) from freqtrade.optimize.backtesting import Backtesting @@ -231,6 +233,8 @@ class LookaheadAnalysis: # first make a single backtest self.fill_full_varholder() + reduce_verbosity_for_bias_tester() + # check if requirements have been met of full_varholder found_signals: int = self.full_varHolder.result['results'].shape[0] + 1 if found_signals >= self.targeted_trade_amount: @@ -251,6 +255,8 @@ class LookaheadAnalysis: break self.analyze_row(idx, result_row) + # Restore verbosity, so it's not too quiet for the next strategy + restore_verbosity_for_bias_tester() # check and report signals if self.current_analysis.total_signals < self.local_config['minimum_trade_amount']: logger.info(f" -> {self.local_config['strategy']} : too few trades. " diff --git a/tests/test_log_setup.py b/tests/test_log_setup.py index a9be24723..2ce06b6b0 100644 --- a/tests/test_log_setup.py +++ b/tests/test_log_setup.py @@ -7,6 +7,8 @@ import pytest from freqtrade.exceptions import OperationalException from freqtrade.loggers import (FTBufferingHandler, FTStdErrStreamHandler, set_loggers, setup_logging, setup_logging_pre) +from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, + restore_verbosity_for_bias_tester) def test_set_loggers() -> None: @@ -128,3 +130,19 @@ def test_set_loggers_journald_importerror(import_fails): match=r'You need the cysystemd python package.*'): setup_logging(config) logger.handlers = orig_handlers + + +def test_reduce_verbosity(): + reduce_verbosity_for_bias_tester() + + assert logging.getLogger('freqtrade.resolvers').level is logging.WARNING + assert logging.getLogger('freqtrade.strategy.hyper').level is logging.WARNING + # base level wasn't changed + assert logging.getLogger('freqtrade').level is logging.INFO + + restore_verbosity_for_bias_tester() + + assert logging.getLogger('freqtrade.resolvers').level is logging.INFO + assert logging.getLogger('freqtrade.strategy.hyper').level is logging.INFO + assert logging.getLogger('freqtrade').level is logging.INFO + # base level wasn't changed From 16b3363970733ffb7ca6503402f20cc16b05a827 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Jun 2023 07:16:06 +0200 Subject: [PATCH 37/55] Fix type problem --- freqtrade/optimize/lookahead_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index e98eebeef..65e9cad3f 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -4,7 +4,7 @@ import pathlib import shutil from copy import deepcopy from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import pandas as pd @@ -48,7 +48,7 @@ class LookaheadAnalysis: self.entry_varHolders: List[VarHolder] = [] self.exit_varHolders: List[VarHolder] = [] - self.exchange = None + self.exchange: Optional[Any] = None # pull variables the scope of the lookahead_analysis-instance self.local_config = deepcopy(config) From 99842402f7d12e2581b68cd18e46b14f3fbf2f55 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Jun 2023 07:18:35 +0200 Subject: [PATCH 38/55] Further reduce unnecessary output --- freqtrade/loggers/set_log_levels.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/loggers/set_log_levels.py b/freqtrade/loggers/set_log_levels.py index da046f439..d666361b6 100644 --- a/freqtrade/loggers/set_log_levels.py +++ b/freqtrade/loggers/set_log_levels.py @@ -31,6 +31,7 @@ def set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None: __BIAS_TESTER_LOGGERS = [ 'freqtrade.resolvers', 'freqtrade.strategy.hyper', + 'freqtrade.configuration.config_validation', ] From 6656740f2120b0c0d07cd4cfa4c175e1506d1446 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Fri, 9 Jun 2023 22:11:30 +0200 Subject: [PATCH 39/55] Moved config overrides to its' own function Added config overrides to dry_run_wallet and max_open_trades to avoid false positives. --- .../optimize/lookahead_analysis_helpers.py | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 0eccf0526..0f2b78e24 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -119,25 +119,22 @@ class LookaheadAnalysisSubFunctions: csv_df.to_csv(config['lookahead_analysis_exportfilename'], index=False) @staticmethod - def initialize_single_lookahead_analysis(strategy_obj: Dict[str, Any], config: Dict[str, Any]): - - logger.info(f"Bias test of {Path(strategy_obj['location']).name} started.") - start = time.perf_counter() - current_instance = LookaheadAnalysis(config, strategy_obj) - current_instance.start() - elapsed = time.perf_counter() - start - logger.info(f"checking look ahead bias via backtests " - f"of {Path(strategy_obj['location']).name} " - f"took {elapsed:.0f} seconds.") - return current_instance - - @staticmethod - def start(config: Config): + def calculate_config_overrides(config: Config): if config['targeted_trade_amount'] < config['minimum_trade_amount']: # this combo doesn't make any sense. raise OperationalException( - "targeted trade amount can't be smaller than minimum trade amount." + "Targeted trade amount can't be smaller than minimum trade amount." ) + if len(config['pairs']) > config['max_open_trades']: + logger.info('Max_open_trades were less than amount of pairs. ' + 'Set max_open_trades to amount of pairs just to avoid false positives.') + config['max_open_trades'] = len(config['pairs']) + + min_dry_run_wallet = 1000000000 + if config['dry_run_wallet'] < min_dry_run_wallet: + logger.info('Dry run wallet was not set to 1 billion, pushing it up there ' + 'just to avoid false positives') + config['dry_run_wallet'] = min_dry_run_wallet # enforce cache to be 'none', shift it to 'none' if not already # (since the default value is 'day') @@ -149,6 +146,24 @@ class LookaheadAnalysisSubFunctions: f"Inside lookahead-analysis it is enforced to be 'none'. " f"Changed it to 'none'") config['backtest_cache'] = 'none' + return config + + @staticmethod + def initialize_single_lookahead_analysis(strategy_obj: Dict[str, Any], config: Dict[str, Any]): + + logger.info(f"Bias test of {Path(strategy_obj['location']).name} started.") + start = time.perf_counter() + current_instance = LookaheadAnalysis(config, strategy_obj) + current_instance.start() + elapsed = time.perf_counter() - start + logger.info(f"Checking look ahead bias via backtests " + f"of {Path(strategy_obj['location']).name} " + f"took {elapsed:.0f} seconds.") + return current_instance + + @staticmethod + def start(config: Config): + config = LookaheadAnalysisSubFunctions.calculate_config_overrides(config) strategy_objs = StrategyResolver.search_all_objects( config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) From 94ca2988a0b58dec3ab69f9c9bd1770bc6227518 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Fri, 9 Jun 2023 23:32:58 +0200 Subject: [PATCH 40/55] updated docs --- docs/lookahead-analysis.md | 46 +++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/docs/lookahead-analysis.md b/docs/lookahead-analysis.md index 22440a6d6..d61acf370 100644 --- a/docs/lookahead-analysis.md +++ b/docs/lookahead-analysis.md @@ -5,8 +5,7 @@ Checking look ahead bias is the bane of any strategy since it is sometimes very but very hard to detect. Backtesting initializes all timestamps at once and calculates all indicators in the beginning. -This means that if you are allowing your indicators (or the libraries that get used) then you would -look into the future and falsify your backtest. +This means that if your indicators or entry/exit signals could look into future candles and falsify your backtest. Lookahead-analysis requires historic data to be available. To learn how to get data for the pairs and exchange you're interested in, @@ -14,13 +13,15 @@ head over to the [Data Downloading](data-download.md) section of the documentati This command is built upon backtesting since it internally chains backtests and pokes at the strategy to provoke it to show look ahead bias. -This is done by looking not at the strategy itself - but at the results it returned. +This is done by not looking at the strategy itself - but at the results it returned. The results are things like changed indicator-values and moved entries/exits compared to the full backtest. You can use commands of [Backtesting](backtesting.md). It also supports the lookahead-analysis of freqai strategies. ---cache is enforced to be "none" +- --cache is forced to "none" +- --max_open_trades is forced to be at least equal to the number of pairs +- --dry_run_wallet is forced to be basically infinite ## Backtesting command reference @@ -46,7 +47,7 @@ optional arguments: #### Summary -Checks a given strategy for look ahead bias via backtest-analysis +Checks a given strategy for look ahead bias via lookahead-analysis Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting and producing false hopes for the one backtesting. @@ -61,19 +62,22 @@ but not realistic. This command is made to try to verify the validity in the form of the aforementioned look ahead bias. #### How does the command work? -It will not look at the strategy or any contents itself but instead will run multiple backtests -by using precisely cut timeranges and analyzing the results each time, comparing to the full timerange. - -At first, it starts a backtest over the whole duration -and then repeats backtests from the same starting point to the respective points to watch. -In addition, it analyzes the dataframes form the overall backtest to the cut ones. - -At the end it will return a result-table in terminal. - -Hint: -If an entry or exit condition is only triggered rarely or the timerange was chosen -so only a few entry conditions are met -then the bias checker is unable to catch the biased entry or exit condition. -In the end it only checks which entry and exit signals have been triggered. - ----Flow chart here for better understanding--- +It will start with a backtest of all pairs to generate a baseline for indicators and entries/exits. +After the backtest ran, it will look if the minimum-trade-amount is met +and if not cancel the lookahead-analysis for this strategy. + +After setting the baseline it will then do additional runs for every entry and exit separately. +When a verification-backtest is done, it will compare the indicators as the signal (either entry or exit) +and report the bias. +After all signals have been verified or falsified a result-table will be generated for the user to see. + +#### Caveats: +- The lookahead-analysis can only verify / falsify the trades it calculated through. +If there was a strategy with signals that were not triggered in the lookahead-analysis +then it will not have it verified that entry/exit signal either. +This could then lead to a false-negative (the strategy will then be reported as non-biased). +- lookahead-analysis has access to everything that backtesting has too. +Please don't provoke any configs like enabling position stacking. +If you decide to do so, +then make doubly sure that you won't ever run out of max_open_trades +amount and neither leftover money in your wallet. From 3523f564bd546298e9d3d4d134ac118df13a5dea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Jun 2023 09:44:08 +0200 Subject: [PATCH 41/55] Improve Log reduction and corresponding test --- freqtrade/loggers/set_log_levels.py | 2 +- tests/test_log_setup.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/freqtrade/loggers/set_log_levels.py b/freqtrade/loggers/set_log_levels.py index d666361b6..7311fa0a0 100644 --- a/freqtrade/loggers/set_log_levels.py +++ b/freqtrade/loggers/set_log_levels.py @@ -50,6 +50,6 @@ def restore_verbosity_for_bias_tester() -> None: Restore verbosity after bias tester. """ logger.info("Restoring log verbosity.") - log_level = logging.getLogger('freqtrade').getEffectiveLevel() + log_level = logging.NOTSET for logger_name in __BIAS_TESTER_LOGGERS: logging.getLogger(logger_name).setLevel(log_level) diff --git a/tests/test_log_setup.py b/tests/test_log_setup.py index 2ce06b6b0..af9c43fbd 100644 --- a/tests/test_log_setup.py +++ b/tests/test_log_setup.py @@ -133,16 +133,17 @@ def test_set_loggers_journald_importerror(import_fails): def test_reduce_verbosity(): + setup_logging_pre() reduce_verbosity_for_bias_tester() - assert logging.getLogger('freqtrade.resolvers').level is logging.WARNING - assert logging.getLogger('freqtrade.strategy.hyper').level is logging.WARNING + assert logging.getLogger('freqtrade.resolvers').getEffectiveLevel() is logging.WARNING + assert logging.getLogger('freqtrade.strategy.hyper').getEffectiveLevel() is logging.WARNING # base level wasn't changed - assert logging.getLogger('freqtrade').level is logging.INFO + assert logging.getLogger('freqtrade').getEffectiveLevel() is logging.INFO restore_verbosity_for_bias_tester() - assert logging.getLogger('freqtrade.resolvers').level is logging.INFO - assert logging.getLogger('freqtrade.strategy.hyper').level is logging.INFO - assert logging.getLogger('freqtrade').level is logging.INFO + assert logging.getLogger('freqtrade.resolvers').getEffectiveLevel() is logging.INFO + assert logging.getLogger('freqtrade.strategy.hyper').getEffectiveLevel() is logging.INFO + assert logging.getLogger('freqtrade').getEffectiveLevel() is logging.INFO # base level wasn't changed From 1da1972c18ebd1b5e624987fba22bf74ac48985c Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 11 Jun 2023 00:18:34 +0200 Subject: [PATCH 42/55] added test for config overrides --- tests/optimize/test_lookahead_analysis.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 476627c57..5d054cfa1 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -20,6 +20,8 @@ def lookahead_conf(default_conf_usdt): default_conf_usdt['strategy_path'] = str( Path(__file__).parent.parent / "strategy/strats/lookahead_bias") default_conf_usdt['strategy'] = 'strategy_test_v3_with_lookahead_bias' + default_conf_usdt['max_open_trades'] = 1 + default_conf_usdt['dry_run_wallet'] = 1000000000 return default_conf_usdt @@ -339,3 +341,13 @@ def test_biased_strategy(lookahead_conf, mocker, caplog, scenario) -> None: # check biased strategy elif scenario == "bias1": assert instance.current_analysis.has_bias + + +def test_config_overrides(lookahead_conf): + lookahead_conf['max_open_trades'] = 0 + lookahead_conf['dry_run_wallet'] = 1 + lookahead_conf['pairs'] = ['BTC/USDT', 'ETH/USDT', 'SOL/USDT'] + lookahead_conf = LookaheadAnalysisSubFunctions.calculate_config_overrides(lookahead_conf) + + assert lookahead_conf['dry_run_wallet'] == 1000000000 + assert lookahead_conf['max_open_trades'] == 3 From 663cfc62111f7572645eede86ea398071ee6e26a Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 11 Jun 2023 22:53:21 +0200 Subject: [PATCH 43/55] fixing tests --- tests/optimize/test_lookahead_analysis.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 5d054cfa1..7678726ae 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -22,7 +22,7 @@ def lookahead_conf(default_conf_usdt): default_conf_usdt['strategy'] = 'strategy_test_v3_with_lookahead_bias' default_conf_usdt['max_open_trades'] = 1 default_conf_usdt['dry_run_wallet'] = 1000000000 - + default_conf_usdt['pairs'] = ['UNITTEST/USDT'] return default_conf_usdt @@ -40,6 +40,10 @@ def test_start_lookahead_analysis(mocker): "strategy_test_v3_with_lookahead_bias", "--strategy-path", str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), + "--pairs", + "UNITTEST/BTC", + "--max-open-trades", + "1" ] pargs = get_args(args) pargs['config'] = None @@ -65,19 +69,22 @@ def test_start_lookahead_analysis(mocker): pargs = get_args(args) pargs['config'] = None with pytest.raises(OperationalException, - match=r"targeted trade amount can't be smaller than .*"): + match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): start_lookahead_analysis(pargs) -def test_lookahead_helper_invalid_config(lookahead_conf, mocker, caplog) -> None: +def test_lookahead_helper_invalid_config(lookahead_conf, caplog) -> None: conf = deepcopy(lookahead_conf) conf['targeted_trade_amount'] = 10 conf['minimum_trade_amount'] = 40 with pytest.raises(OperationalException, - match=r"targeted trade amount can't be smaller than .*"): + match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): LookaheadAnalysisSubFunctions.start(conf) + +def test_lookahead_helper_no_strategy_defined(lookahead_conf, caplog): conf = deepcopy(lookahead_conf) + conf['pairs'] = ['UNITTEST/USDT'] del conf['strategy'] with pytest.raises(OperationalException, match=r"No Strategy specified"): From 11d7e7925eca75d89c2b4d3d061d42e5539a7f60 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Jun 2023 20:34:18 +0200 Subject: [PATCH 44/55] Fix random test failures --- tests/test_log_setup.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_log_setup.py b/tests/test_log_setup.py index af9c43fbd..bd3399615 100644 --- a/tests/test_log_setup.py +++ b/tests/test_log_setup.py @@ -135,15 +135,16 @@ def test_set_loggers_journald_importerror(import_fails): def test_reduce_verbosity(): setup_logging_pre() reduce_verbosity_for_bias_tester() + prior_level = logging.getLogger('freqtrade').getEffectiveLevel() - assert logging.getLogger('freqtrade.resolvers').getEffectiveLevel() is logging.WARNING - assert logging.getLogger('freqtrade.strategy.hyper').getEffectiveLevel() is logging.WARNING + assert logging.getLogger('freqtrade.resolvers').getEffectiveLevel() == logging.WARNING + assert logging.getLogger('freqtrade.strategy.hyper').getEffectiveLevel() == logging.WARNING # base level wasn't changed - assert logging.getLogger('freqtrade').getEffectiveLevel() is logging.INFO + assert logging.getLogger('freqtrade').getEffectiveLevel() == prior_level restore_verbosity_for_bias_tester() - assert logging.getLogger('freqtrade.resolvers').getEffectiveLevel() is logging.INFO - assert logging.getLogger('freqtrade.strategy.hyper').getEffectiveLevel() is logging.INFO - assert logging.getLogger('freqtrade').getEffectiveLevel() is logging.INFO + assert logging.getLogger('freqtrade.resolvers').getEffectiveLevel() == prior_level + assert logging.getLogger('freqtrade.strategy.hyper').getEffectiveLevel() == prior_level + assert logging.getLogger('freqtrade').getEffectiveLevel() == prior_level # base level wasn't changed From ca88cac08bec8baaac08582a1f4b6adfa85dc3b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Jun 2023 06:39:00 +0200 Subject: [PATCH 45/55] Remove unused code file --- .../backtest_lookahead_bias_checker.py | 252 ------------------ 1 file changed, 252 deletions(-) delete mode 100755 freqtrade/strategy/backtest_lookahead_bias_checker.py diff --git a/freqtrade/strategy/backtest_lookahead_bias_checker.py b/freqtrade/strategy/backtest_lookahead_bias_checker.py deleted file mode 100755 index 2e5ef4165..000000000 --- a/freqtrade/strategy/backtest_lookahead_bias_checker.py +++ /dev/null @@ -1,252 +0,0 @@ -import copy -import pathlib -import shutil -from copy import deepcopy -from datetime import datetime, timedelta, timezone - -from pandas import DataFrame - -from freqtrade.configuration import TimeRange -from freqtrade.data.history import get_timerange -from freqtrade.exchange import timeframe_to_minutes -from freqtrade.optimize.backtesting import Backtesting - - -class VarHolder: - timerange: TimeRange - data: DataFrame - indicators: DataFrame - result: DataFrame - compared: DataFrame - from_dt: datetime - to_dt: datetime - compared_dt: datetime - - -class Analysis: - def __init__(self): - self.total_signals = 0 - self.false_entry_signals = 0 - self.false_exit_signals = 0 - self.false_indicators = [] - self.has_bias = False - - total_signals: int - false_entry_signals: int - false_exit_signals: int - - false_indicators: list - has_bias: bool - - -class BacktestLookaheadBiasChecker: - - def __init__(self): - self.exportfilename = None - self.strategy_obj = None - self.current_analysis = None - self.local_config = None - self.full_varHolder = None - - self.entry_varHolder = None - self.exit_varHolder = None - self.entry_varHolders = [] - self.exit_varHolders = [] - self.backtesting = None - self.minimum_trade_amount = None - self.targeted_trade_amount = None - self.failed_bias_check = True - - @staticmethod - def dt_to_timestamp(dt): - timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) - return timestamp - - @staticmethod - def get_result(backtesting, processed): - min_date, max_date = get_timerange(processed) - - result = backtesting.backtest( - processed=deepcopy(processed), - start_date=min_date, - end_date=max_date - ) - return result - - @staticmethod - def report_signal(result, column_name, checked_timestamp): - df = result['results'] - row_count = df[column_name].shape[0] - - if row_count == 0: - return False - else: - - df_cut = df[(df[column_name] == checked_timestamp)] - if df_cut[column_name].shape[0] == 0: - # print("did NOT find the same signal in column " + column_name + - # " at timestamp " + str(checked_timestamp)) - return False - else: - return True - return False - - # analyzes two data frames with processed indicators and shows differences between them. - def analyze_indicators(self, full_vars, cut_vars, current_pair): - # extract dataframes - cut_df = cut_vars.indicators[current_pair] - full_df = full_vars.indicators[current_pair] - - # cut longer dataframe to length of the shorter - full_df_cut = full_df[ - (full_df.date == cut_vars.compared_dt) - ].reset_index(drop=True) - cut_df_cut = cut_df[ - (cut_df.date == cut_vars.compared_dt) - ].reset_index(drop=True) - - # compare dataframes - if full_df_cut.shape[0] != 0: - if cut_df_cut.shape[0] != 0: - compare_df = full_df_cut.compare(cut_df_cut) - - if compare_df.shape[0] > 0: - for col_name, values in compare_df.items(): - col_idx = compare_df.columns.get_loc(col_name) - compare_df_row = compare_df.iloc[0] - # compare_df now comprises tuples with [1] having either 'self' or 'other' - if 'other' in col_name[1]: - continue - self_value = compare_df_row[col_idx] - other_value = compare_df_row[col_idx + 1] - - # output differences - if self_value != other_value: - - if not self.current_analysis.false_indicators.__contains__(col_name[0]): - self.current_analysis.false_indicators.append(col_name[0]) - print(f"=> found look ahead bias in indicator {col_name[0]}. " + - f"{str(self_value)} != {str(other_value)}") - - def prepare_data(self, varHolder, pairs_to_load): - - # purge previous data - abs_folder_path = pathlib.Path("user_data/models/uniqe-id").resolve() - # remove folder and its contents - if pathlib.Path.exists(abs_folder_path): - shutil.rmtree(abs_folder_path) - - prepare_data_config = copy.deepcopy(self.local_config) - prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varHolder.from_dt)) + "-" + - str(self.dt_to_timestamp(varHolder.to_dt))) - prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load - - self.backtesting = Backtesting(prepare_data_config) - self.backtesting._set_strategy(self.backtesting.strategylist[0]) - varHolder.data, varHolder.timerange = self.backtesting.load_bt_data() - self.backtesting.load_bt_data_detail() - - varHolder.indicators = self.backtesting.strategy.advise_all_indicators(varHolder.data) - varHolder.result = self.get_result(self.backtesting, varHolder.indicators) - - def start(self, config, strategy_obj: dict, args) -> None: - - # deepcopy so we can change the pairs for the 2ndary runs - # and not worry about another strategy to check after. - self.local_config = deepcopy(config) - self.local_config['strategy_list'] = [strategy_obj['name']] - self.current_analysis = Analysis() - self.minimum_trade_amount = args['minimum_trade_amount'] - self.targeted_trade_amount = args['targeted_trade_amount'] - self.exportfilename = args['exportfilename'] - self.strategy_obj = strategy_obj - - # first make a single backtest - self.full_varHolder = VarHolder() - - # define datetime in human-readable format - parsed_timerange = TimeRange.parse_timerange(config['timerange']) - - if parsed_timerange.startdt is None: - self.full_varHolder.from_dt = datetime.utcfromtimestamp(0) - else: - self.full_varHolder.from_dt = parsed_timerange.startdt - - if parsed_timerange.stopdt is None: - self.full_varHolder.to_dt = datetime.now() - else: - self.full_varHolder.to_dt = parsed_timerange.stopdt - - self.prepare_data(self.full_varHolder, self.local_config['pairs']) - - found_signals: int = self.full_varHolder.result['results'].shape[0] + 1 - if found_signals >= self.targeted_trade_amount: - print(f"Found {found_signals} trades, calculating {self.targeted_trade_amount} trades.") - elif self.targeted_trade_amount >= found_signals >= self.minimum_trade_amount: - print(f"Only found {found_signals} trades. Calculating all available trades.") - else: - print(f"found {found_signals} trades " - f"which is less than minimum_trade_amount {self.minimum_trade_amount}. " - f"Cancelling this backtest lookahead bias test.") - return - - # now we loop through all entry signals - # starting from the same datetime to avoid miss-reports of bias - for idx, result_row in self.full_varHolder.result['results'].iterrows(): - if self.current_analysis.total_signals == self.targeted_trade_amount: - break - - # if force-sold, ignore this signal since here it will unconditionally exit. - if result_row.close_date == self.dt_to_timestamp(self.full_varHolder.to_dt): - continue - - self.current_analysis.total_signals += 1 - - self.entry_varHolder = VarHolder() - self.exit_varHolder = VarHolder() - self.entry_varHolders.append(self.entry_varHolder) - self.exit_varHolders.append(self.exit_varHolder) - - self.entry_varHolder.from_dt = self.full_varHolder.from_dt - self.entry_varHolder.compared_dt = result_row['open_date'] - # to_dt needs +1 candle since it won't buy on the last candle - self.entry_varHolder.to_dt = (result_row['open_date'] + - timedelta(minutes=timeframe_to_minutes( - self.local_config['timeframe']))) - - self.prepare_data(self.entry_varHolder, [result_row['pair']]) - - # to_dt needs +1 candle since it will always exit/force-exit trades on the last candle - self.exit_varHolder.from_dt = self.full_varHolder.from_dt - self.exit_varHolder.to_dt = (result_row['close_date'] + - timedelta(minutes=timeframe_to_minutes( - self.local_config['timeframe']))) - self.exit_varHolder.compared_dt = result_row['close_date'] - - self.prepare_data(self.exit_varHolder, [result_row['pair']]) - - # register if buy signal is broken - if not self.report_signal( - self.entry_varHolder.result, "open_date", self.entry_varHolder.compared_dt): - self.current_analysis.false_entry_signals += 1 - - # register if buy or sell signal is broken - if not self.report_signal( - self.exit_varHolder.result, "close_date", self.exit_varHolder.compared_dt): - self.current_analysis.false_exit_signals += 1 - - if len(self.entry_varHolders) >= 10: - pass - # check if the indicators themselves contain biased data - self.analyze_indicators(self.full_varHolder, self.entry_varHolder, result_row['pair']) - self.analyze_indicators(self.full_varHolder, self.exit_varHolder, result_row['pair']) - - if (self.current_analysis.false_entry_signals > 0 or - self.current_analysis.false_exit_signals > 0 or - len(self.current_analysis.false_indicators) > 0): - print(" => " + self.local_config['strategy_list'][0] + ": bias detected!") - self.current_analysis.has_bias = True - else: - print(self.local_config['strategy_list'][0] + ": no bias detected") - - self.failed_bias_check = False From ac36ba65926f6ee25c7960113ed1fbd9bd7dcd58 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Jun 2023 20:12:11 +0200 Subject: [PATCH 46/55] Improve arguments file formatting --- freqtrade/commands/arguments.py | 16 ++++++---------- freqtrade/commands/strategy_utils_commands.py | 0 2 files changed, 6 insertions(+), 10 deletions(-) mode change 100755 => 100644 freqtrade/commands/strategy_utils_commands.py diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index af5f0a470..a2d2f8c5c 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -124,10 +124,6 @@ ARGS_LOOKAHEAD_ANALYSIS = ARGS_BACKTEST + ["minimum_trade_amount", "lookahead_analysis_exportfilename"] -# + ["target_trades", "minimum_trades", -# "target_trades", "exportfilename"] -# will be added when the base version works. - class Arguments: """ Arguments Class. Manage the arguments received by the cli @@ -460,14 +456,14 @@ class Arguments: 'files to the current version', parents=[_common_parser]) strategy_updater_cmd.set_defaults(func=start_strategy_update) - self._build_args(optionlist=ARGS_STRATEGY_UPDATER, - parser=strategy_updater_cmd) + self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd) # Add lookahead_analysis subcommand - lookahead_analayis_cmd = \ - subparsers.add_parser('lookahead-analysis', - help="checks for potential look ahead bias", - parents=[_common_parser, _strategy_parser]) + lookahead_analayis_cmd = subparsers.add_parser( + 'lookahead-analysis', + help="Check for potential look ahead bias.", + parents=[_common_parser, _strategy_parser]) + lookahead_analayis_cmd.set_defaults(func=start_lookahead_analysis) self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS, diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py old mode 100755 new mode 100644 From ad74e65673d7d528ef07f49d56350077dd00bfa3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Jun 2023 20:23:47 +0200 Subject: [PATCH 47/55] Simplify configuration setup --- freqtrade/configuration/configuration.py | 6 ++---- freqtrade/optimize/lookahead_analysis_helpers.py | 1 + tests/optimize/test_lookahead_analysis.py | 11 +++++------ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 5bbbf301d..a64eaa0ca 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -300,10 +300,8 @@ class Configuration: self._args_to_config(config, argname='hyperoptexportfilename', logstring='Using hyperopt file: {}') - if self.args.get('lookahead_analysis_exportfilename'): - if self.args["lookahead_analysis_exportfilename"] is not None: - self._args_to_config(config, argname='lookahead_analysis_exportfilename', - logstring='saving lookahead analysis results into {} ...') + self._args_to_config(config, argname='lookahead_analysis_exportfilename', + logstring='Saving lookahead analysis results into {} ...') self._args_to_config(config, argname='epochs', logstring='Parameter --epochs detected ... ' diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 0f2b78e24..22cbbfa6b 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -15,6 +15,7 @@ logger = logging.getLogger(__name__) class LookaheadAnalysisSubFunctions: + @staticmethod def text_table_lookahead_analysis_instances( config: Dict[str, Any], diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 7678726ae..d2e6fbbe1 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -114,12 +114,11 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf analysis.false_entry_signals = 4 analysis.false_exit_signals = 3 - strategy_obj = \ - { - 'name': "strategy_test_v3_with_lookahead_bias", - 'location': PurePosixPath(lookahead_conf['strategy_path'], - f"{lookahead_conf['strategy']}.py") - } + strategy_obj = { + 'name': "strategy_test_v3_with_lookahead_bias", + 'location': PurePosixPath(lookahead_conf['strategy_path'], + f"{lookahead_conf['strategy']}.py") + } instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.current_analysis = analysis From 964bf76469414137a27300112423b3787d651d40 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Jun 2023 20:42:26 +0200 Subject: [PATCH 48/55] Invert parameters for initialize_single_lookahead_analysis otherwise their order is reversed before calling LookaheadAnalysis for no good reason --- freqtrade/optimize/lookahead_analysis_helpers.py | 4 ++-- tests/optimize/test_lookahead_analysis.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 22cbbfa6b..702eee774 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -150,7 +150,7 @@ class LookaheadAnalysisSubFunctions: return config @staticmethod - def initialize_single_lookahead_analysis(strategy_obj: Dict[str, Any], config: Dict[str, Any]): + def initialize_single_lookahead_analysis(config: Config, strategy_obj: Dict[str, Any]): logger.info(f"Bias test of {Path(strategy_obj['location']).name} started.") start = time.perf_counter() @@ -186,7 +186,7 @@ class LookaheadAnalysisSubFunctions: if strategy_obj['name'] == strat and strategy_obj not in strategy_list: lookaheadAnalysis_instances.append( LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( - strategy_obj, config)) + config, strategy_obj)) break # report the results diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index d2e6fbbe1..8539db7f3 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -291,7 +291,7 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() -def test_initialize_single_lookahead_analysis(lookahead_conf, mocker): +def test_initialize_single_lookahead_analysis(lookahead_conf, mocker, caplog): mocker.patch('freqtrade.data.history.get_timerange', get_timerange) mocker.patch(f'{EXMS}.get_fee', return_value=0.0) mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) @@ -303,9 +303,17 @@ def test_initialize_single_lookahead_analysis(lookahead_conf, mocker): lookahead_conf['timeframe'] = '5m' lookahead_conf['timerange'] = '20180119-20180122' - strategy_obj = {'name': "strategy_test_v3_with_lookahead_bias"} + start_mock = mocker.patch('freqtrade.optimize.lookahead_analysis.LookaheadAnalysis.start') + strategy_obj = { + 'name': "strategy_test_v3_with_lookahead_bias", + 'location': Path(lookahead_conf['strategy_path'], f"{lookahead_conf['strategy']}.py") + } + + instance = LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( + lookahead_conf, strategy_obj) + assert log_has_re(r"Bias test of .* started\.", caplog) + assert start_mock.call_count == 1 - instance = LookaheadAnalysis(lookahead_conf, strategy_obj) assert instance.strategy_obj['name'] == "strategy_test_v3_with_lookahead_bias" From b3ef024e9e6c402edf279254b52e14ccf18e28a2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Jun 2023 20:43:05 +0200 Subject: [PATCH 49/55] Don't use PurePosixPath --- tests/optimize/test_lookahead_analysis.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 8539db7f3..3c6a5ad6d 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -1,6 +1,6 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument from copy import deepcopy -from pathlib import Path, PurePosixPath +from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pytest @@ -73,7 +73,7 @@ def test_start_lookahead_analysis(mocker): start_lookahead_analysis(pargs) -def test_lookahead_helper_invalid_config(lookahead_conf, caplog) -> None: +def test_lookahead_helper_invalid_config(lookahead_conf) -> None: conf = deepcopy(lookahead_conf) conf['targeted_trade_amount'] = 10 conf['minimum_trade_amount'] = 40 @@ -82,7 +82,7 @@ def test_lookahead_helper_invalid_config(lookahead_conf, caplog) -> None: LookaheadAnalysisSubFunctions.start(conf) -def test_lookahead_helper_no_strategy_defined(lookahead_conf, caplog): +def test_lookahead_helper_no_strategy_defined(lookahead_conf): conf = deepcopy(lookahead_conf) conf['pairs'] = ['UNITTEST/USDT'] del conf['strategy'] @@ -91,7 +91,7 @@ def test_lookahead_helper_no_strategy_defined(lookahead_conf, caplog): LookaheadAnalysisSubFunctions.start(conf) -def test_lookahead_helper_start(lookahead_conf, mocker, caplog) -> None: +def test_lookahead_helper_start(lookahead_conf, mocker) -> None: single_mock = MagicMock() text_table_mock = MagicMock() mocker.patch.multiple( @@ -107,7 +107,7 @@ def test_lookahead_helper_start(lookahead_conf, mocker, caplog) -> None: text_table_mock.reset_mock() -def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf, caplog): +def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf): analysis = Analysis() analysis.has_bias = True analysis.total_signals = 5 @@ -116,8 +116,7 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf strategy_obj = { 'name': "strategy_test_v3_with_lookahead_bias", - 'location': PurePosixPath(lookahead_conf['strategy_path'], - f"{lookahead_conf['strategy']}.py") + 'location': Path(lookahead_conf['strategy_path'], f"{lookahead_conf['strategy']}.py") } instance = LookaheadAnalysis(lookahead_conf, strategy_obj) @@ -192,7 +191,7 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): strategy_obj1 = { 'name': "strat1", - 'location': PurePosixPath("file1.py"), + 'location': Path("file1.py"), } instance1 = LookaheadAnalysis(lookahead_conf, strategy_obj1) @@ -237,7 +236,7 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): strategy_obj2 = { 'name': "strat1", - 'location': PurePosixPath("file1.py"), + 'location': Path("file1.py"), } instance2 = LookaheadAnalysis(lookahead_conf, strategy_obj2) @@ -275,7 +274,7 @@ def test_lookahead_helper_export_to_csv(lookahead_conf): strategy_obj3 = { 'name': "strat3", - 'location': PurePosixPath("file3.py"), + 'location': Path("file3.py"), } instance3 = LookaheadAnalysis(lookahead_conf, strategy_obj3) From 2cd9043c51e88b9ccdb6c15235eb0bb5657626a5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Jun 2023 06:44:55 +0200 Subject: [PATCH 50/55] Make documentation discoverable / linked --- docs/lookahead-analysis.md | 38 +++++++++++++++++++------------------- mkdocs.yml | 1 + 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/docs/lookahead-analysis.md b/docs/lookahead-analysis.md index d61acf370..cd1c74d13 100644 --- a/docs/lookahead-analysis.md +++ b/docs/lookahead-analysis.md @@ -1,4 +1,5 @@ # Lookahead analysis + This page explains how to validate your strategy in terms of look ahead bias. Checking look ahead bias is the bane of any strategy since it is sometimes very easy to introduce backtest bias - @@ -11,19 +12,18 @@ Lookahead-analysis requires historic data to be available. To learn how to get data for the pairs and exchange you're interested in, head over to the [Data Downloading](data-download.md) section of the documentation. -This command is built upon backtesting -since it internally chains backtests and pokes at the strategy to provoke it to show look ahead bias. +This command is built upon backtesting since it internally chains backtests and pokes at the strategy to provoke it to show look ahead bias. This is done by not looking at the strategy itself - but at the results it returned. -The results are things like changed indicator-values and moved entries/exits compared to the full backtest. +The results are things like changed indicator-values and moved entries/exits compared to the full backtest. You can use commands of [Backtesting](backtesting.md). It also supports the lookahead-analysis of freqai strategies. -- --cache is forced to "none" -- --max_open_trades is forced to be at least equal to the number of pairs -- --dry_run_wallet is forced to be basically infinite +- `--cache` is forced to "none". +- `--max-open-trades` is forced to be at least equal to the number of pairs. +- `--dry-run-wallet` is forced to be basically infinite. -## Backtesting command reference +## Lookahead-analysis command reference ``` usage: freqtrade lookahead-analysis [-h] [-v] [-V] @@ -45,13 +45,14 @@ optional arguments: Use this filename to save your lookahead-analysis-results to a csv file ``` +### Summary -#### Summary Checks a given strategy for look ahead bias via lookahead-analysis Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting and producing false hopes for the one backtesting. -#### Introduction: +### Introduction + Many strategies - without the programmer knowing - have fallen prey to look ahead bias. Any backtest will populate the full dataframe including all time stamps at the beginning. @@ -61,9 +62,10 @@ but not realistic. This command is made to try to verify the validity in the form of the aforementioned look ahead bias. -#### How does the command work? +### How does the command work? + It will start with a backtest of all pairs to generate a baseline for indicators and entries/exits. -After the backtest ran, it will look if the minimum-trade-amount is met +After the backtest ran, it will look if the `minimum-trade-amount` is met and if not cancel the lookahead-analysis for this strategy. After setting the baseline it will then do additional runs for every entry and exit separately. @@ -71,13 +73,11 @@ When a verification-backtest is done, it will compare the indicators as the sign and report the bias. After all signals have been verified or falsified a result-table will be generated for the user to see. -#### Caveats: -- The lookahead-analysis can only verify / falsify the trades it calculated through. -If there was a strategy with signals that were not triggered in the lookahead-analysis -then it will not have it verified that entry/exit signal either. +### Caveats + +- `lookahead-analysis` can only verify / falsify the trades it calculated through. +If there was a strategy with signals that were not triggered during the lookahead-analysis, then it will not have it verified that entry/exit signal either. This could then lead to a false-negative (the strategy will then be reported as non-biased). -- lookahead-analysis has access to everything that backtesting has too. +- `lookahead-analysis` has access to everything that backtesting has too. Please don't provoke any configs like enabling position stacking. -If you decide to do so, -then make doubly sure that you won't ever run out of max_open_trades -amount and neither leftover money in your wallet. +If you decide to do so, then make doubly sure that you won't ever run out of `max_open_trades` amount and neither leftover money in your wallet. diff --git a/mkdocs.yml b/mkdocs.yml index 3f9e8a880..815a10419 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,6 +22,7 @@ nav: - Web Hook: webhook-config.md - Data Downloading: data-download.md - Backtesting: backtesting.md + - Lookahead analysis: lookahead-analysis.md - Hyperopt: hyperopt.md - FreqAI: - Introduction: freqai.md From 1b86bf8a1db0d304417dc8e110d9205c37a6fd2a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Jun 2023 06:58:34 +0200 Subject: [PATCH 51/55] Don't include non-used parameters in command structure --- freqtrade/commands/arguments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index a2d2f8c5c..b0da8fa9d 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -119,9 +119,9 @@ NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"] -ARGS_LOOKAHEAD_ANALYSIS = ARGS_BACKTEST + ["minimum_trade_amount", - "targeted_trade_amount", - "lookahead_analysis_exportfilename"] +ARGS_LOOKAHEAD_ANALYSIS = [ + a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", 'cache') + ] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"] class Arguments: From 2c7aa9f721e181b3461cf95cd8eca1f85aeca299 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Jun 2023 08:37:38 +0200 Subject: [PATCH 52/55] Update doc wording --- docs/lookahead-analysis.md | 57 +++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/docs/lookahead-analysis.md b/docs/lookahead-analysis.md index cd1c74d13..9d57de779 100644 --- a/docs/lookahead-analysis.md +++ b/docs/lookahead-analysis.md @@ -26,25 +26,43 @@ It also supports the lookahead-analysis of freqai strategies. ## Lookahead-analysis command reference ``` -usage: freqtrade lookahead-analysis [-h] [-v] [-V] - [--minimum-trade-amount INT] - [--targeted-trade-amount INT] - [--lookahead-analysis-exportfilename PATH] - -optional arguments: - -h, --help show this help message and exit +usage: freqtrade lookahead-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [-s NAME] + [--strategy-path PATH] + [--recursive-strategy-search] + [--freqaimodel NAME] + [--freqaimodel-path PATH] [-i TIMEFRAME] + [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}] + [--max-open-trades INT] + [--stake-amount STAKE_AMOUNT] + [--fee FLOAT] [-p PAIRS [PAIRS ...]] + [--enable-protections] + [--dry-run-wallet DRY_RUN_WALLET] + [--timeframe-detail TIMEFRAME_DETAIL] + [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] + [--export {none,trades,signals}] + [--export-filename PATH] + [--breakdown {day,week,month} [{day,week,month} ...]] + [--cache {none,day,week,month}] + [--freqai-backtest-live-models] + [--minimum-trade-amount INT] + [--targeted-trade-amount INT] + [--lookahead-analysis-exportfilename LOOKAHEAD_ANALYSIS_EXPORTFILENAME] + +options: --minimum-trade-amount INT - Override the value of the `minimum_trade_amount` configuration - setting - Requires `--targeted-trade-amount` to be larger or equal to --minimum-trade-amount. - (default: 10) + Minimum trade amount for lookahead-analysis --targeted-trade-amount INT - Override the value of the `minimum_trade_amount` configuration - (default: 20) - --lookahead-analysis-exportfilename PATH - Use this filename to save your lookahead-analysis-results to a csv file + Targeted trade amount for lookahead analysis + --lookahead-analysis-exportfilename LOOKAHEAD_ANALYSIS_EXPORTFILENAME + Use this csv-filename to store lookahead-analysis- + results ``` +!!! Note "" + The above Output was reduced to options `lookahead-analysis` adds on top of regular backtesting commands. + ### Summary Checks a given strategy for look ahead bias via lookahead-analysis @@ -69,15 +87,14 @@ After the backtest ran, it will look if the `minimum-trade-amount` is met and if not cancel the lookahead-analysis for this strategy. After setting the baseline it will then do additional runs for every entry and exit separately. -When a verification-backtest is done, it will compare the indicators as the signal (either entry or exit) -and report the bias. +When a verification-backtest is done, it will compare the indicators as the signal (either entry or exit) and report the bias. After all signals have been verified or falsified a result-table will be generated for the user to see. ### Caveats -- `lookahead-analysis` can only verify / falsify the trades it calculated through. -If there was a strategy with signals that were not triggered during the lookahead-analysis, then it will not have it verified that entry/exit signal either. -This could then lead to a false-negative (the strategy will then be reported as non-biased). +- `lookahead-analysis` can only verify / falsify the trades it calculated and verified. +If the strategy has many different signals / signal types, it's up to you to select appropriate parameters to ensure that all signals have triggered at least once. Not triggered signals will not have been verified. +This could lead to a false-negative (the strategy will then be reported as non-biased). - `lookahead-analysis` has access to everything that backtesting has too. Please don't provoke any configs like enabling position stacking. If you decide to do so, then make doubly sure that you won't ever run out of `max_open_trades` amount and neither leftover money in your wallet. From 34e7e3efea14e707b89bc95d02f7b725dceea9eb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Jun 2023 08:40:09 +0200 Subject: [PATCH 53/55] Simplify imports --- freqtrade/optimize/lookahead_analysis.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 65e9cad3f..889e43375 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -1,9 +1,8 @@ -import copy import logging -import pathlib import shutil from copy import deepcopy from datetime import datetime, timedelta, timezone +from pathlib import Path from typing import Any, Dict, List, Optional import pandas as pd @@ -134,14 +133,13 @@ class LookaheadAnalysis: # purge previous data if the freqai model is defined # (to be sure nothing is carried over from older backtests) path_to_current_identifier = ( - pathlib.Path(f"{self.local_config['user_data_dir']}" - "/models/" - f"{self.local_config['freqai']['identifier']}").resolve()) + Path(f"{self.local_config['user_data_dir']}/models/" + f"{self.local_config['freqai']['identifier']}").resolve()) # remove folder and its contents - if pathlib.Path.exists(path_to_current_identifier): + if Path.exists(path_to_current_identifier): shutil.rmtree(path_to_current_identifier) - prepare_data_config = copy.deepcopy(self.local_config) + prepare_data_config = deepcopy(self.local_config) prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + str(self.dt_to_timestamp(varholder.to_dt))) prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load From 6bb75f0dd498abeb32c1688ff1283b99fe2ad108 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Jun 2023 10:12:36 +0200 Subject: [PATCH 54/55] Simplify import if only one element is used --- freqtrade/optimize/lookahead_analysis.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 889e43375..32528909d 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Dict, List, Optional -import pandas as pd +from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.data.history import get_timerange @@ -20,10 +20,10 @@ logger = logging.getLogger(__name__) class VarHolder: timerange: TimeRange - data: pd.DataFrame - indicators: pd.DataFrame - result: pd.DataFrame - compared: pd.DataFrame + data: DataFrame + indicators: Dict[str, DataFrame] + result: DataFrame + compared: DataFrame from_dt: datetime to_dt: datetime compared_dt: datetime @@ -63,7 +63,7 @@ class LookaheadAnalysis: return timestamp @staticmethod - def get_result(backtesting: Backtesting, processed: pd.DataFrame): + def get_result(backtesting: Backtesting, processed: DataFrame): min_date, max_date = get_timerange(processed) result = backtesting.backtest( @@ -92,8 +92,8 @@ class LookaheadAnalysis: # analyzes two data frames with processed indicators and shows differences between them. def analyze_indicators(self, full_vars: VarHolder, cut_vars: VarHolder, current_pair): # extract dataframes - cut_df = cut_vars.indicators[current_pair] - full_df = full_vars.indicators[current_pair] + cut_df: DataFrame = cut_vars.indicators[current_pair] + full_df: DataFrame = full_vars.indicators[current_pair] # cut longer dataframe to length of the shorter full_df_cut = full_df[ @@ -127,7 +127,7 @@ class LookaheadAnalysis: f"{col_name[0]}. " f"{str(self_value)} != {str(other_value)}") - def prepare_data(self, varholder: VarHolder, pairs_to_load: List[pd.DataFrame]): + def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]): if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: # purge previous data if the freqai model is defined From bf872e8ed48dda59ebe704f7a059488c0aa72eef Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Jun 2023 14:25:46 +0200 Subject: [PATCH 55/55] Simplify comparison depth --- freqtrade/optimize/lookahead_analysis.py | 51 ++++++++++++------------ 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 32528909d..dcc1088b3 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -43,7 +43,7 @@ class LookaheadAnalysis: def __init__(self, config: Dict[str, Any], strategy_obj: Dict): self.failed_bias_check = True - self.full_varHolder = VarHolder + self.full_varHolder = VarHolder() self.entry_varHolders: List[VarHolder] = [] self.exit_varHolders: List[VarHolder] = [] @@ -90,7 +90,7 @@ class LookaheadAnalysis: return False # analyzes two data frames with processed indicators and shows differences between them. - def analyze_indicators(self, full_vars: VarHolder, cut_vars: VarHolder, current_pair): + def analyze_indicators(self, full_vars: VarHolder, cut_vars: VarHolder, current_pair: str): # extract dataframes cut_df: DataFrame = cut_vars.indicators[current_pair] full_df: DataFrame = full_vars.indicators[current_pair] @@ -103,29 +103,30 @@ class LookaheadAnalysis: (cut_df.date == cut_vars.compared_dt) ].reset_index(drop=True) - # compare dataframes - if full_df_cut.shape[0] != 0: - if cut_df_cut.shape[0] != 0: - compare_df = full_df_cut.compare(cut_df_cut) - - if compare_df.shape[0] > 0: - for col_name, values in compare_df.items(): - col_idx = compare_df.columns.get_loc(col_name) - compare_df_row = compare_df.iloc[0] - # compare_df now comprises tuples with [1] having either 'self' or 'other' - if 'other' in col_name[1]: - continue - self_value = compare_df_row[col_idx] - other_value = compare_df_row[col_idx + 1] - - # output differences - if self_value != other_value: - - if not self.current_analysis.false_indicators.__contains__(col_name[0]): - self.current_analysis.false_indicators.append(col_name[0]) - logger.info(f"=> found look ahead bias in indicator " - f"{col_name[0]}. " - f"{str(self_value)} != {str(other_value)}") + # check if dataframes are not empty + if full_df_cut.shape[0] != 0 and cut_df_cut.shape[0] != 0: + + # compare dataframes + compare_df = full_df_cut.compare(cut_df_cut) + + if compare_df.shape[0] > 0: + for col_name, values in compare_df.items(): + col_idx = compare_df.columns.get_loc(col_name) + compare_df_row = compare_df.iloc[0] + # compare_df now comprises tuples with [1] having either 'self' or 'other' + if 'other' in col_name[1]: + continue + self_value = compare_df_row[col_idx] + other_value = compare_df_row[col_idx + 1] + + # output differences + if self_value != other_value: + + if not self.current_analysis.false_indicators.__contains__(col_name[0]): + self.current_analysis.false_indicators.append(col_name[0]) + logger.info(f"=> found look ahead bias in indicator " + f"{col_name[0]}. " + f"{str(self_value)} != {str(other_value)}") def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]):