Merge pull request #5 from Corax-CoLAB/ftttlcache-jobs-15723335757504013040

Use FtTTLCache for Background Jobs

freqtrade/rpc/api_server/api_background_tasks.py

@@ -15,18 +15,33 @@ router = APIRouter()
 
 @router.get("/background", response_model=list[BackgroundTaskStatus])
 def background_job_list():
-    return [
-        {
-            "job_id": jobid,
-            "job_category": job["category"],
-            "status": job["status"],
-            "running": job["is_running"],
-            "progress": job.get("progress"),
-            "progress_tasks": job.get("progress_tasks"),
-            "error": job.get("error", None),
-        }
-        for jobid, job in ApiBG.jobs.items()
-    ]
+    try:
+        return [
+            {
+                "job_id": jobid,
+                "job_category": job["category"],
+                "status": job["status"],
+                "running": job["is_running"],
+                "progress": job.get("progress"),
+                "progress_tasks": job.get("progress_tasks"),
+                "error": job.get("error", None),
+            }
+            for jobid, job in ApiBG.jobs.items()
+        ]
+    except RuntimeError:
+        # Dictionary changed size during iteration
+        return [
+            {
+                "job_id": jobid,
+                "job_category": job["category"],
+                "status": job["status"],
+                "running": job["is_running"],
+                "progress": job.get("progress"),
+                "progress_tasks": job.get("progress_tasks"),
+                "error": job.get("error", None),
+            }
+            for jobid, job in ApiBG.jobs.items()
+        ]
 
 
 @router.get("/background/{jobid}", response_model=BackgroundTaskStatus)

freqtrade/rpc/api_server/api_download_data.py

@@ -1,4 +1,5 @@
 import logging
+import time
 from copy import deepcopy
 
 from fastapi import APIRouter, BackgroundTasks, Depends
@@ -27,6 +28,7 @@ def __run_download(job_id: str, config_loc: Config):
 
         with FtNoDBContext():
             exchange = get_exchange(config_loc)
+            last_refresh = [0.0]
 
             def ft_callback(task) -> None:
                 ApiBG.jobs[job_id]["progress_tasks"][str(task.id)] = {
@@ -34,6 +36,10 @@ def __run_download(job_id: str, config_loc: Config):
                     "total": task.total,
                     "description": task.description,
                 }
+                if time.time() - last_refresh[0] > 60:
+                    if job := ApiBG.jobs.get(job_id):
+                        ApiBG.jobs[job_id] = job
+                        last_refresh[0] = time.time()
 
             pt = get_progress_tracker(ft_callback=ft_callback)
 
@@ -44,7 +50,9 @@ def __run_download(job_id: str, config_loc: Config):
         ApiBG.jobs[job_id]["error"] = str(e)
         ApiBG.jobs[job_id]["status"] = "failed"
     finally:
-        ApiBG.jobs[job_id]["is_running"] = False
+        if job := ApiBG.jobs.get(job_id):
+            job["is_running"] = False
+            ApiBG.jobs[job_id] = job
         ApiBG.download_data_running = False
 
 

freqtrade/rpc/api_server/api_pairlists.py

@@ -66,7 +66,9 @@ def __run_pairlist(job_id: str, config_loc: Config):
         ApiBG.jobs[job_id]["error"] = str(e)
         ApiBG.jobs[job_id]["status"] = "failed"
     finally:
-        ApiBG.jobs[job_id]["is_running"] = False
+        if job := ApiBG.jobs.get(job_id):
+            job["is_running"] = False
+            ApiBG.jobs[job_id] = job
         ApiBG.pairlist_running = False
 
 

freqtrade/rpc/api_server/webserver_bgwork.py

@@ -4,6 +4,7 @@ from uuid import uuid4
 from typing_extensions import TypedDict
 
 from freqtrade.exchange.exchange import Exchange
+from freqtrade.util import FtTTLCache
 
 
 class ProgressTask(TypedDict):
@@ -38,7 +39,7 @@ class ApiBG:
     # Generic background jobs
 
     # TODO: Change this to FtTTLCache
-    jobs: dict[str, JobsContainer] = {}
+    jobs: dict[str, JobsContainer] = FtTTLCache(maxsize=1000, ttl=3600)  # type: ignore
     # Pairlist evaluate things
     pairlist_running: bool = False
     download_data_running: bool = False