From 5f03641b7d49d0f3bb30ff011120a4e1531c6096 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Tue, 6 Jan 2026 14:45:05 -0800 Subject: [PATCH] Refactor watchlist auto-scanning to use heartbeat thread Replaces the timer-based automatic watchlist scanning with a background heartbeat thread for improved reliability and robust retry logic. The new system handles scheduling, conflict detection, and error recovery, while UI countdown updates are decoupled from actual scheduling. --- web_server.py | 122 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 34 deletions(-) diff --git a/web_server.py b/web_server.py index 2e8f7a73..e2ded352 100644 --- a/web_server.py +++ b/web_server.py @@ -251,7 +251,8 @@ wishlist_timer_lock = threading.Lock() # Thread safety for timer operations # --- Automatic Watchlist Scanning Infrastructure --- # Server-side timer system for automatic watchlist scanning (mirrors wishlist pattern for consistency) -watchlist_auto_timer = None # threading.Timer for scheduling next auto-scanning +watchlist_scheduler_thread = None # background thread +watchlist_stop_event = threading.Event() watchlist_auto_scanning = False # Flag to prevent concurrent auto-scanning watchlist_auto_scanning_timestamp = 0 # Timestamp when scanning started (for stuck detection) watchlist_next_run_time = 0 # Timestamp when next auto-scanning is scheduled (for countdown display) @@ -17458,47 +17459,99 @@ watchlist_scan_state = { 'error': None } +def _watchlist_heartbeat_loop(): + """ + Background heartbeat loop that triggers watchlist scanning every 24 hours. + Allows for robust retry logic when conflicts occur with the wishlist processor. + """ + print("💓 [Auto-Watchlist] Heartbeat scheduler started") + + # Initial delay of 5 minutes + if watchlist_stop_event.wait(300): + return + + while not watchlist_stop_event.is_set(): + try: + # Run the processing logic + # Returns: 'completed', 'skipped_conflict', 'skipped_idle', 'skipped_active' + result = _process_watchlist_scan_automatically() + + if result == 'skipped_conflict': + # If conflict with wishlist, wait 10 minutes and retry + print("âŗ [Auto-Watchlist] Pausing 10 minutes before retry due to wishlist conflict") + + # Update UI timer + with watchlist_timer_lock: + watchlist_next_run_time = time.time() + 600.0 + + if watchlist_stop_event.wait(600): + break + else: + # Standard 24 hour cycle for success or other skips + with watchlist_timer_lock: + watchlist_next_run_time = time.time() + 86400.0 + + if watchlist_stop_event.wait(86400): + break + + except Exception as e: + print(f"❌ [Auto-Watchlist] Error in heartbeat loop: {e}") + # Wait 10 minutes on error before retrying to avoid spamming + with watchlist_timer_lock: + watchlist_next_run_time = time.time() + 600.0 + + if watchlist_stop_event.wait(600): + break + def start_watchlist_auto_scanning(): - """Start automatic watchlist scanning with 5-minute initial delay (Timer-based like wishlist)""" - global watchlist_auto_timer, watchlist_next_run_time + """Start automatic watchlist scanning with robust heartbeat scheduler.""" + global watchlist_scheduler_thread, watchlist_next_run_time + + # Reset stop event + watchlist_stop_event.clear() print("🚀 [Auto-Watchlist] Initializing automatic watchlist scanning...") with watchlist_timer_lock: - # Stop any existing timer to prevent duplicates - if watchlist_auto_timer is not None: - watchlist_auto_timer.cancel() + # Stop any existing thread + if watchlist_scheduler_thread is not None and watchlist_scheduler_thread.is_alive(): + print("⚠ī¸ Watchlist scheduler already running") + return - print("🔄 Starting automatic watchlist scanning system (5 minute initial delay)") + print("🔄 Starting automatic watchlist heartbeat system (5 minute initial delay)") watchlist_next_run_time = time.time() + 300.0 # Set timestamp for countdown display - watchlist_auto_timer = threading.Timer(300.0, _process_watchlist_scan_automatically) # 5 minutes - watchlist_auto_timer.daemon = True - watchlist_auto_timer.start() - print(f"✅ [Auto-Watchlist] Timer started successfully - will trigger in 5 minutes") + + watchlist_scheduler_thread = threading.Thread(target=_watchlist_heartbeat_loop, daemon=True, name="WatchlistHeartbeat") + watchlist_scheduler_thread.start() + print(f"✅ [Auto-Watchlist] Heartbeat thread started successfully") def stop_watchlist_auto_scanning(): - """Stop automatic watchlist scanning and cleanup timer.""" - global watchlist_auto_timer, watchlist_auto_scanning + """Stop automatic watchlist scanning and cleanup thread.""" + global watchlist_scheduler_thread, watchlist_auto_scanning, watchlist_auto_scanning_timestamp, watchlist_next_run_time with watchlist_timer_lock: - if watchlist_auto_timer is not None: - watchlist_auto_timer.cancel() - watchlist_auto_timer = None + # Signal thread to stop + watchlist_stop_event.set() + + if watchlist_scheduler_thread is not None: + watchlist_scheduler_thread = None print("⏚ī¸ Stopped automatic watchlist scanning") watchlist_auto_scanning = False watchlist_auto_scanning_timestamp = 0 + watchlist_next_run_time = 0 def schedule_next_watchlist_scan(): - """Schedule next automatic watchlist scan in 24 hours.""" - global watchlist_auto_timer, watchlist_next_run_time + """ + Update the UI timestamp for the next run. + NOTE: The actual scheduling is now handled by _watchlist_heartbeat_loop. + This function exists to update the UI countdown. + """ + global watchlist_next_run_time with watchlist_timer_lock: - print("⏰ Scheduling next automatic watchlist scan in 24 hours") + print("⏰ [UI Update] Updating next watchlist scan timestamp (handled by heartbeat)") watchlist_next_run_time = time.time() + 86400.0 # Set timestamp for countdown display - watchlist_auto_timer = threading.Timer(86400.0, _process_watchlist_scan_automatically) # 24 hours - watchlist_auto_timer.daemon = True - watchlist_auto_timer.start() def _process_watchlist_scan_automatically(): """Main automatic scanning logic that runs in background thread.""" @@ -17510,19 +17563,19 @@ def _process_watchlist_scan_automatically(): with watchlist_timer_lock: if watchlist_auto_scanning: print("⚠ī¸ Watchlist auto-scanning already running, skipping.") - schedule_next_watchlist_scan() - return + # We return 'skipped_active' so heartbeat knows we're busy but alive + # Heartbeat will wait 24h as per normal cycle since it's already running + schedule_next_watchlist_scan() + return 'skipped_active' # Check if wishlist processing is currently running (using smart detection) if is_wishlist_actually_processing(): print("đŸŽĩ Wishlist processing in progress, rescheduling watchlist scan for 10 minutes from now") - # Smart retry: don't wait 24 hours, just wait 10 minutes and try again - global watchlist_auto_timer, watchlist_next_run_time - watchlist_next_run_time = time.time() + 600.0 # Set timestamp for countdown display - watchlist_auto_timer = threading.Timer(600.0, _process_watchlist_scan_automatically) # 10 minutes - watchlist_auto_timer.daemon = True - watchlist_auto_timer.start() - return + # Just update UI timer. Heartbeat loop will handle the 10m wait and retry. + global watchlist_next_run_time + with watchlist_timer_lock: + watchlist_next_run_time = time.time() + 600.0 + return 'skipped_conflict' # Set flag and timestamp import time @@ -17546,7 +17599,7 @@ def _process_watchlist_scan_automatically(): watchlist_auto_scanning = False watchlist_auto_scanning_timestamp = 0 schedule_next_watchlist_scan() - return + return 'skipped_idle' if not spotify_client or not spotify_client.is_authenticated(): print("ℹī¸ [Auto-Watchlist] Spotify client not available or not authenticated.") @@ -17804,6 +17857,8 @@ def _process_watchlist_scan_automatically(): if total_added_to_wishlist > 0: add_activity_item("👁ī¸", "Watchlist Scan Complete", f"{total_added_to_wishlist} new tracks added to wishlist", "Now") + return 'completed' + except Exception as e: print(f"❌ Error in automatic watchlist scan: {e}") import traceback @@ -17813,11 +17868,10 @@ def _process_watchlist_scan_automatically(): watchlist_scan_state['error'] = str(e) finally: - # Always reset flag and schedule next scan + # Always reset flag with watchlist_timer_lock: watchlist_auto_scanning = False watchlist_auto_scanning_timestamp = 0 - schedule_next_watchlist_scan() print("✅ Automatic watchlist scanning initialized")