consolidate and clean code

docs/freqai-running.md

@@ -79,11 +79,11 @@ To change your **features**, you **must** set a new `identifier` in the config t
 
 To save the models generated during a particular backtest so that you can start a live deployment from one of them instead of training a new model, you must set `save_backtest_models` to `True` in the config.
 
-### Backtest live models
+### Backtest live collected predictions
 
 FreqAI allow you to reuse live historic predictions through the backtest parameter `--freqai-backtest-live-models`. This can be useful when you want to reuse predictions generated in dry/run for comparison or other study.
 
-The `--timerange` parameter must not be informed, as it will be automatically calculated through the data in historic predictions file.
+The `--timerange` parameter must not be informed, as it will be automatically calculated through the data in the historic predictions file.
 
 
 ### Downloading data to cover the full backtest period

freqtrade/freqai/data_kitchen.py

@@ -75,7 +75,6 @@ class FreqaiDataKitchen:
         self.training_features_list: List = []
         self.model_filename: str = ""
         self.backtesting_results_path = Path()
-        self.backtesting_h5_data: HDFStore = {}
         self.backtest_predictions_folder: str = "backtesting_predictions"
         self.live = live
         self.pair = pair
@@ -456,28 +455,6 @@ class FreqaiDataKitchen:
         # print(tr_training_list, tr_backtesting_list)
         return tr_training_list_timerange, tr_backtesting_list_timerange
 
-    # def split_timerange_live_models(
-    #     self
-    # ) -> Tuple[list, list]:
-
-    #     tr_backtesting_list_timerange = []
-    #     asset = self.pair.split("/")[0]
-    #     if asset not in self.backtest_live_models_data["assets_end_dates"]:
-    #         raise OperationalException(
-    #             f"Model not available for pair {self.pair}. "
-    #             "Please, try again after removing this pair from the configuration file."
-    #         )
-    #     asset_data = self.backtest_live_models_data["assets_end_dates"][asset]
-    #     backtesting_timerange = self.backtest_live_models_data["backtesting_timerange"]
-    #     model_end_dates = [x for x in asset_data]
-    #     model_end_dates.append(backtesting_timerange.stopts)
-    #     model_end_dates.sort()
-    #     for index, item in enumerate(model_end_dates):
-    #         if len(model_end_dates) > (index + 1):
-    #             tr_to_add = TimeRange("date", "date", item, model_end_dates[index + 1])
-    #             tr_backtesting_list_timerange.append(tr_to_add)
-
-    #     return tr_backtesting_list_timerange, tr_backtesting_list_timerange
 
     def slice_dataframe(self, timerange: TimeRange, df: DataFrame) -> DataFrame:
         """

freqtrade/freqai/freqai_interface.py

@@ -55,7 +55,6 @@ class IFreqaiModel(ABC):
     def __init__(self, config: Config) -> None:
 
         self.config = config
-        self.metadata: Dict[str, Any] = {}
         self.assert_config(self.config)
         self.freqai_info: Dict[str, Any] = config["freqai"]
         self.data_split_parameters: Dict[str, Any] = config.get("freqai", {}).get(
@@ -102,7 +101,7 @@ class IFreqaiModel(ABC):
         self.get_corr_dataframes: bool = True
         self._threads: List[threading.Thread] = []
         self._stop_event = threading.Event()
-        self.metadata = self.dd.load_global_metadata_from_disk()
+        self.metadata: Dict[str, Any] = self.dd.load_global_metadata_from_disk()
         self.data_provider: Optional[DataProvider] = None
         self.max_system_threads = max(int(psutil.cpu_count() * 2 - 2), 1)
 
@@ -148,18 +147,13 @@ class IFreqaiModel(ABC):
         # the concatenated results for the full backtesting period back to the strategy.
         elif not self.follow_mode:
             self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
-            if self.dk.backtest_live_models:
-                logger.info(
-                    "Backtesting using historic predictions (live models)")
-            else:
-                logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
-            dataframe = self.dk.use_strategy_to_populate_indicators(
-                strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
-            )
             if not self.config.get("freqai_backtest_live_models", False):
+                logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
                 dk = self.start_backtesting(dataframe, metadata, self.dk)
                 dataframe = dk.remove_features_from_df(dk.return_dataframe)
             else:
+                logger.info(
+                    "Backtesting using historic predictions (live models)")
                 dk = self.start_backtesting_from_historic_predictions(
                     dataframe, metadata, self.dk)
                 dataframe = dk.return_dataframe
@@ -167,7 +161,6 @@ class IFreqaiModel(ABC):
         self.clean_up()
         if self.live:
             self.inference_timer('stop', metadata["pair"])
-            self.set_start_dry_live_date(dataframe)
 
         return dataframe
 
@@ -336,27 +329,6 @@ class IFreqaiModel(ABC):
 
         return dk
 
-    def start_backtesting_from_historic_predictions(
-        self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen
-    ) -> FreqaiDataKitchen:
-        """
-        :param dataframe: DataFrame = strategy passed dataframe
-        :param metadata: Dict = pair metadata
-        :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only
-        :return:
-            FreqaiDataKitchen = Data management/analysis tool associated to present pair only
-        """
-        pair = metadata["pair"]
-        dk.return_dataframe = dataframe
-        saved_dataframe = self.dd.historic_predictions[pair]
-        columns_to_drop = list(set(saved_dataframe.columns).intersection(
-            dk.return_dataframe.columns))
-        dk.return_dataframe = dk.return_dataframe.drop(columns=list(columns_to_drop))
-        dk.return_dataframe = pd.merge(
-            dk.return_dataframe, saved_dataframe, how='left', left_on='date', right_on="date_pred")
-        # dk.return_dataframe = dk.return_dataframe[saved_dataframe.columns].fillna(0)
-        return dk
-
     def start_live(
         self, dataframe: DataFrame, metadata: dict, strategy: IStrategy, dk: FreqaiDataKitchen
     ) -> FreqaiDataKitchen:
@@ -665,6 +637,8 @@ class IFreqaiModel(ABC):
         self.dd.historic_predictions[pair] = pred_df
         hist_preds_df = self.dd.historic_predictions[pair]
 
+        self.set_start_dry_live_date(pred_df)
+
         for label in hist_preds_df.columns:
             if hist_preds_df[label].dtype == object:
                 continue
@@ -913,6 +887,27 @@ class IFreqaiModel(ABC):
                 pd.to_datetime(live_dataframe.tail(1)["date"].values[0]).timestamp())
             self.update_metadata(metadata)
 
+    def start_backtesting_from_historic_predictions(
+        self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen
+    ) -> FreqaiDataKitchen:
+        """
+        :param dataframe: DataFrame = strategy passed dataframe
+        :param metadata: Dict = pair metadata
+        :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only
+        :return:
+            FreqaiDataKitchen = Data management/analysis tool associated to present pair only
+        """
+        pair = metadata["pair"]
+        dk.return_dataframe = dataframe
+        saved_dataframe = self.dd.historic_predictions[pair]
+        columns_to_drop = list(set(saved_dataframe.columns).intersection(
+            dk.return_dataframe.columns))
+        dk.return_dataframe = dk.return_dataframe.drop(columns=list(columns_to_drop))
+        dk.return_dataframe = pd.merge(
+            dk.return_dataframe, saved_dataframe, how='left', left_on='date', right_on="date_pred")
+        # dk.return_dataframe = dk.return_dataframe[saved_dataframe.columns].fillna(0)
+        return dk
+
     # Following methods which are overridden by user made prediction models.
     # See freqai/prediction_models/CatboostPredictionModel.py for an example.