Merge pull request #10827 from freqtrade/fix/freqai-zeros

fix: guarantee crash resiliency, as long as users reload bot gracefully

build_helpers/schema.json

@@ -1383,6 +1383,11 @@
           "type": "string",
           "default": "example"
         },
+        "wait_for_training_iteration_on_reload": {
+          "description": "Wait for the next training iteration to complete after /reload or ctrl+c.",
+          "type": "boolean",
+          "default": true
+        },
         "feature_parameters": {
           "description": "The parameters used to engineer the feature set",
           "type": "object",

docs/freqai-parameter-table.md

@@ -22,6 +22,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file. <br> **Datatype:** Boolean. <br> Default: `False`
 | `data_kitchen_thread_count` | <br> Designate the number of threads you want to use for data processing (outlier methods, normalization, etc.). This has no impact on the number of threads used for training. If user does not set it (default), FreqAI will use max number of threads - 2 (leaving 1 physical core available for Freqtrade bot and FreqUI) <br> **Datatype:** Positive integer.
 | `activate_tensorboard` | <br> Indicate whether or not to activate tensorboard for the tensorboard enabled modules (currently Reinforcment Learning, XGBoost, Catboost, and PyTorch). Tensorboard needs Torch installed, which means you will need the torch/RL docker image or you need to answer "yes" to the install question about whether or not you wish to install Torch. <br> **Datatype:** Boolean. <br> Default: `True`.
+| `wait_for_training_iteration_on_reload` | <br> When using /reload or ctrl-c, wait for the current training iteration to finish before completing graceful shutdown. If set to `False`, FreqAI will break the current training iteration, allowing you to shutdown gracefully more quickly, but you will lose your current training iteration. <br> **Datatype:** Boolean. <br> Default: `True`.
 
 ### Feature parameters
 

freqtrade/configuration/config_schema.py

@@ -995,6 +995,13 @@ CONF_SCHEMA = {
                     "type": "string",
                     "default": "example",
                 },
+                "wait_for_training_iteration_on_reload": {
+                    "description": (
+                        "Wait for the next training iteration to complete after /reload or ctrl+c."
+                    ),
+                    "type": "boolean",
+                    "default": True,
+                },
                 "feature_parameters": {
                     "description": "The parameters used to engineer the feature set",
                     "type": "object",

freqtrade/freqai/freqai_interface.py

@@ -185,6 +185,7 @@ class IFreqaiModel(ABC):
         Callback for Subclasses to override to include logic for shutting down resources
         when SIGINT is sent.
         """
+        self.dd.save_historic_predictions_to_disk()
         return
 
     def shutdown(self):
@@ -198,9 +199,16 @@ class IFreqaiModel(ABC):
         self.data_provider = None
         self._on_stop()
 
-        logger.info("Waiting on Training iteration")
-        for _thread in self._threads:
-            _thread.join()
+        if self.freqai_info.get("wait_for_training_iteration_on_reload", True):
+            logger.info("Waiting on Training iteration")
+            for _thread in self._threads:
+                _thread.join()
+        else:
+            logger.warning(
+                "Breaking current training iteration because "
+                "you set wait_for_training_iteration_on_reload to "
+                " False."
+            )
 
     def start_scanning(self, *args, **kwargs) -> None:
         """