From c94b1d2f5bee24d3896e42ebc7d7906f8b952af8 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 3 Jan 2026 22:20:36 -0800 Subject: [PATCH] Add hybrid Soulseek/YouTube download orchestration - TESTING Introduces a DownloadOrchestrator class to route downloads between Soulseek and YouTube based on user-configurable modes (Soulseek only, YouTube only, Hybrid with fallback). Updates web server and UI to support new download source settings, including hybrid mode options and YouTube confidence threshold. Refactors YouTube client for thread-safe download management and bot detection bypass. Ensures quality filtering is skipped for YouTube results and improves file matching and post-processing logic for YouTube downloads. --- config/settings.py | 5 + core/download_orchestrator.py | 304 ++++++++++++++++++++++++++++++++++ core/youtube_client.py | 278 +++++++++++++++++++------------ web_server.py | 127 +++++++++++--- webui/index.html | 34 ++++ webui/static/script.js | 27 ++- 6 files changed, 649 insertions(+), 126 deletions(-) create mode 100644 core/download_orchestrator.py diff --git a/config/settings.py b/config/settings.py index c37cd439..43327079 100644 --- a/config/settings.py +++ b/config/settings.py @@ -145,6 +145,11 @@ class ConfigManager: "download_path": "./downloads", "transfer_path": "./Transfer" }, + "download_source": { + "mode": "soulseek", # Options: "soulseek", "youtube", "hybrid" + "hybrid_primary": "soulseek", # Which source to try first in hybrid mode + "youtube_min_confidence": 0.65 # Minimum confidence for YouTube matches + }, "listenbrainz": { "token": "" }, diff --git a/core/download_orchestrator.py b/core/download_orchestrator.py new file mode 100644 index 00000000..5dff8143 --- /dev/null +++ b/core/download_orchestrator.py @@ -0,0 +1,304 @@ +""" +Download Orchestrator +Routes downloads between Soulseek and YouTube based on configuration. + +Supports three modes: +- Soulseek Only: Traditional behavior +- YouTube Only: YouTube-exclusive downloads +- Hybrid: Try primary source first, fallback to secondary if it fails +""" + +import asyncio +from typing import List, Optional, Tuple +from pathlib import Path + +from utils.logging_config import get_logger +from config.settings import config_manager +from core.soulseek_client import SoulseekClient, TrackResult, AlbumResult, DownloadStatus +from core.youtube_client import YouTubeClient + +logger = get_logger("download_orchestrator") + + +class DownloadOrchestrator: + """ + Orchestrates downloads between Soulseek and YouTube based on user preferences. + + Acts as a drop-in replacement for SoulseekClient by exposing the same async interface. + Routes requests to the appropriate client(s) based on configured mode. + """ + + def __init__(self): + """Initialize orchestrator with both clients""" + self.soulseek = SoulseekClient() + self.youtube = YouTubeClient() + + # Load mode from config + self.mode = config_manager.get('download_source.mode', 'soulseek') + self.hybrid_primary = config_manager.get('download_source.hybrid_primary', 'soulseek') + self.youtube_min_confidence = config_manager.get('download_source.youtube_min_confidence', 0.65) + + logger.info(f"🎛️ Download Orchestrator initialized - Mode: {self.mode}") + if self.mode == 'hybrid': + logger.info(f" Primary source: {self.hybrid_primary}") + + def reload_settings(self): + """Reload settings from config (call after settings change)""" + self.mode = config_manager.get('download_source.mode', 'soulseek') + self.hybrid_primary = config_manager.get('download_source.hybrid_primary', 'soulseek') + self.youtube_min_confidence = config_manager.get('download_source.youtube_min_confidence', 0.65) + + logger.info(f"🔄 Download Orchestrator settings reloaded - Mode: {self.mode}") + + def is_configured(self) -> bool: + """ + Check if orchestrator is configured and ready to use. + + Returns True if at least one download source is configured. + """ + if self.mode == 'soulseek': + return self.soulseek.is_configured() + elif self.mode == 'youtube': + return self.youtube.is_configured() + elif self.mode == 'hybrid': + # In hybrid mode, at least one source must be configured + return self.soulseek.is_configured() or self.youtube.is_configured() + + return False + + async def check_connection(self) -> bool: + """ + Test if download sources are accessible. + + Returns True if the configured source(s) are reachable. + """ + if self.mode == 'soulseek': + return await self.soulseek.check_connection() + elif self.mode == 'youtube': + return await self.youtube.check_connection() + elif self.mode == 'hybrid': + # In hybrid mode, check both sources + soulseek_ok = await self.soulseek.check_connection() + youtube_ok = await self.youtube.check_connection() + + logger.info(f" Soulseek: {'✅' if soulseek_ok else '❌'} | YouTube: {'✅' if youtube_ok else '❌'}") + + # At least one must be available + return soulseek_ok or youtube_ok + + return False + + async def search(self, query: str, timeout: int = None, progress_callback=None) -> Tuple[List[TrackResult], List[AlbumResult]]: + """ + Search for tracks using configured source(s). + + Args: + query: Search query + timeout: Search timeout (for Soulseek) + progress_callback: Progress callback (for Soulseek) + + Returns: + Tuple of (track_results, album_results) + """ + if self.mode == 'soulseek': + logger.info(f"🔍 Searching Soulseek: {query}") + return await self.soulseek.search(query, timeout, progress_callback) + + elif self.mode == 'youtube': + logger.info(f"🔍 Searching YouTube: {query}") + return await self.youtube.search(query, timeout, progress_callback) + + elif self.mode == 'hybrid': + # Try primary source first + if self.hybrid_primary == 'soulseek': + logger.info(f"🔍 Hybrid search - trying Soulseek first: {query}") + tracks, albums = await self.soulseek.search(query, timeout, progress_callback) + + # If Soulseek found good results, return them + if tracks: + logger.info(f"✅ Soulseek found {len(tracks)} tracks") + return (tracks, albums) + + # Otherwise, try YouTube as fallback + logger.info(f"🔄 Soulseek found nothing, trying YouTube fallback") + return await self.youtube.search(query, timeout, progress_callback) + + else: # YouTube first + logger.info(f"🔍 Hybrid search - trying YouTube first: {query}") + tracks, albums = await self.youtube.search(query, timeout, progress_callback) + + # If YouTube found good results, return them + if tracks: + logger.info(f"✅ YouTube found {len(tracks)} tracks") + return (tracks, albums) + + # Otherwise, try Soulseek as fallback + logger.info(f"🔄 YouTube found nothing, trying Soulseek fallback") + return await self.soulseek.search(query, timeout, progress_callback) + + # Fallback: empty results + return ([], []) + + async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]: + """ + Download a track using the appropriate client. + + Args: + username: Username (or "youtube" for YouTube) + filename: Filename or YouTube video ID + file_size: File size estimate + + Returns: + download_id: Unique download ID for tracking + """ + # Detect which client to use based on username + if username == 'youtube': + logger.info(f"📥 Downloading from YouTube: {filename}") + return await self.youtube.download(username, filename, file_size) + else: + logger.info(f"📥 Downloading from Soulseek: {filename}") + return await self.soulseek.download(username, filename, file_size) + + async def get_all_downloads(self) -> List[DownloadStatus]: + """ + Get all active downloads from all sources. + + Returns: + List of DownloadStatus objects + """ + # Get downloads from both sources + soulseek_downloads = await self.soulseek.get_all_downloads() + youtube_downloads = await self.youtube.get_all_downloads() + + # Combine and return + return soulseek_downloads + youtube_downloads + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + """ + Get status of a specific download. + + Args: + download_id: Download ID to query + + Returns: + DownloadStatus object or None if not found + """ + # Try Soulseek first + status = await self.soulseek.get_download_status(download_id) + if status: + return status + + # Try YouTube + status = await self.youtube.get_download_status(download_id) + if status: + return status + + return None + + async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool: + """ + Cancel an active download. + + Args: + download_id: Download ID to cancel + username: Username hint (optional) + remove: Whether to remove from active downloads + + Returns: + True if cancelled successfully + """ + # If username is provided, route directly + if username == 'youtube': + return await self.youtube.cancel_download(download_id, username, remove) + elif username: + return await self.soulseek.cancel_download(download_id, username, remove) + + # Otherwise, try both sources + soulseek_cancelled = await self.soulseek.cancel_download(download_id, username, remove) + if soulseek_cancelled: + return True + + youtube_cancelled = await self.youtube.cancel_download(download_id, username, remove) + return youtube_cancelled + + async def signal_download_completion(self, download_id: str, username: str, remove: bool = True) -> bool: + """ + Signal that a download has completed (Soulseek only). + + Args: + download_id: Download ID + username: Username + remove: Whether to remove from active downloads + + Returns: + True if successful + """ + # This is Soulseek-specific, so only call on Soulseek client + return await self.soulseek.signal_download_completion(download_id, username, remove) + + async def clear_all_completed_downloads(self) -> bool: + """ + Clear all completed downloads from both sources. + + Returns: + True if successful + """ + soulseek_cleared = await self.soulseek.clear_all_completed_downloads() + youtube_cleared = True # YouTube auto-removes completed downloads + + return soulseek_cleared and youtube_cleared + + # ===== Soulseek-specific methods (for backwards compatibility) ===== + # These are internal methods that some parts of the codebase use directly + + async def _make_request(self, method: str, endpoint: str, **kwargs): + """ + Proxy to SoulseekClient._make_request for backwards compatibility. + This is a Soulseek-specific internal method. + + Args: + method: HTTP method + endpoint: API endpoint + **kwargs: Additional request parameters + + Returns: + API response + """ + return await self.soulseek._make_request(method, endpoint, **kwargs) + + async def _make_direct_request(self, method: str, endpoint: str, **kwargs): + """ + Proxy to SoulseekClient._make_direct_request for backwards compatibility. + This is a Soulseek-specific internal method. + + Args: + method: HTTP method + endpoint: API endpoint + **kwargs: Additional request parameters + + Returns: + API response + """ + return await self.soulseek._make_direct_request(method, endpoint, **kwargs) + + async def clear_all_searches(self) -> bool: + """ + Clear all searches (Soulseek-specific). + + Returns: + True if successful + """ + return await self.soulseek.clear_all_searches() + + async def maintain_search_history_with_buffer(self, keep_searches: int = 50, trigger_threshold: int = 200) -> bool: + """ + Maintain search history (Soulseek-specific). + + Args: + keep_searches: Number of searches to keep + trigger_threshold: Threshold to trigger cleanup + + Returns: + True if successful + """ + return await self.soulseek.maintain_search_history_with_buffer(keep_searches, trigger_threshold) diff --git a/core/youtube_client.py b/core/youtube_client.py index 14cd6934..c5282a0b 100644 --- a/core/youtube_client.py +++ b/core/youtube_client.py @@ -16,6 +16,7 @@ import re import platform import asyncio import uuid +import threading from typing import List, Optional, Dict, Any, Tuple from dataclasses import dataclass from pathlib import Path @@ -96,10 +97,17 @@ class YouTubeClient: Provides search, matching, and download capabilities with full Spotify metadata integration. """ - def __init__(self, download_path: str = "./downloads/youtube"): + def __init__(self, download_path: str = None): + # Use Soulseek download path for consistency (post-processing expects files here) + from config.settings import config_manager + if download_path is None: + download_path = config_manager.get('soulseek.download_path', './downloads') + self.download_path = Path(download_path) self.download_path.mkdir(parents=True, exist_ok=True) + logger.info(f"📁 YouTube client using download path: {self.download_path}") + # Initialize production matching engine for parity with Soulseek self.matching_engine = MusicMatchingEngine() logger.info("✅ Initialized production MusicMatchingEngine") @@ -112,9 +120,9 @@ class YouTubeClient: # Download queue management (mirrors Soulseek's download tracking) # Maps download_id -> download_info dict self.active_downloads: Dict[str, Dict[str, Any]] = {} - self._download_lock = asyncio.Lock() + self._download_lock = threading.Lock() # Use threading.Lock for thread safety - # Configure yt-dlp options + # Configure yt-dlp options with bot detection bypass self.download_opts = { 'format': 'bestaudio/best', 'outtmpl': str(self.download_path / '%(title)s.%(ext)s'), @@ -127,6 +135,15 @@ class YouTubeClient: 'preferredquality': '320', }], 'progress_hooks': [self._progress_hook], # Track download progress + # Bot detection bypass options + 'extractor_args': { + 'youtube': { + 'player_client': ['android', 'web'], # Try multiple clients + 'skip': ['hls', 'dash'], # Skip problematic formats + } + }, + 'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'age_limit': None, # Don't skip age-restricted } # Track current download progress (mirrors Soulseek transfer tracking) @@ -179,6 +196,14 @@ class YouTubeClient: 'quiet': True, 'no_warnings': True, 'extract_flat': True, # Don't download, just extract info + # Bot detection bypass + 'extractor_args': { + 'youtube': { + 'player_client': ['android', 'web'], + 'skip': ['hls', 'dash'], + } + }, + 'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', } with yt_dlp.YoutubeDL(ydl_opts) as ydl: @@ -245,15 +270,16 @@ class YouTubeClient: else: percent = 0 - # Update active downloads dictionary (thread-safe update) - if self.current_download_id in self.active_downloads: - download_info = self.active_downloads[self.current_download_id] - download_info['state'] = 'Downloading' - download_info['progress'] = round(percent, 1) - download_info['transferred'] = downloaded - download_info['size'] = total - download_info['speed'] = int(speed) - download_info['time_remaining'] = int(eta) if eta > 0 else None + # Update active downloads dictionary (thread-safe update with lock) + with self._download_lock: + if self.current_download_id in self.active_downloads: + download_info = self.active_downloads[self.current_download_id] + download_info['state'] = 'InProgress, Downloading' # Match Soulseek state format + download_info['progress'] = round(percent, 1) + download_info['transferred'] = downloaded + download_info['size'] = total + download_info['speed'] = int(speed) + download_info['time_remaining'] = int(eta) if eta > 0 else None # Also update current_download_progress for legacy compatibility self.current_download_progress = { @@ -271,21 +297,23 @@ class YouTubeClient: self.progress_callback(self.current_download_progress) elif status == 'finished': - # Update to postprocessing state - if self.current_download_id in self.active_downloads: - self.active_downloads[self.current_download_id]['state'] = 'Postprocessing' - self.active_downloads[self.current_download_id]['progress'] = 100.0 + # Download finished, ffmpeg is converting to MP3 + # Keep state as 'InProgress, Downloading' - the download thread will set final state + with self._download_lock: + if self.current_download_id in self.active_downloads: + self.active_downloads[self.current_download_id]['progress'] = 95.0 # Almost done (converting) self.current_download_progress['status'] = 'postprocessing' - self.current_download_progress['percent'] = 100.0 + self.current_download_progress['percent'] = 95.0 if self.progress_callback: self.progress_callback(self.current_download_progress) elif status == 'error': - # Mark as error - if self.current_download_id in self.active_downloads: - self.active_downloads[self.current_download_id]['state'] = 'Errored' + # Mark as error (thread-safe) + with self._download_lock: + if self.current_download_id in self.active_downloads: + self.active_downloads[self.current_download_id]['state'] = 'Errored' self.current_download_progress['status'] = 'error' if self.progress_callback: @@ -525,6 +553,14 @@ class YouTubeClient: 'no_warnings': True, 'extract_flat': False, 'default_search': 'ytsearch', + # Bot detection bypass (same as download options) + 'extractor_args': { + 'youtube': { + 'player_client': ['android', 'web'], + 'skip': ['hls', 'dash'], + } + }, + 'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', } with yt_dlp.YoutubeDL(ydl_opts) as ydl: @@ -716,13 +752,16 @@ class YouTubeClient: """ Download YouTube video as audio (async, Soulseek-compatible interface). + Returns download_id immediately and runs download in background thread. + Monitor via get_download_status() or get_all_downloads(). + Args: username: Ignored for YouTube (always "youtube") filename: Encoded as "video_id||title" from search results file_size: Ignored for YouTube (kept for interface compatibility) Returns: - download_id: Unique ID for tracking this download, or None if failed to start + download_id: Unique ID for tracking this download """ try: # Parse filename to extract video_id @@ -740,12 +779,12 @@ class YouTubeClient: download_id = str(uuid.uuid4()) # Initialize download info in active downloads - async with self._download_lock: + with self._download_lock: self.active_downloads[download_id] = { 'id': download_id, - 'filename': title, + 'filename': filename, # Keep original encoded format for context matching! 'username': 'youtube', - 'state': 'Initializing', + 'state': 'Initializing', # Soulseek-style states 'progress': 0.0, 'size': file_size or 0, 'transferred': 0, @@ -755,52 +794,108 @@ class YouTubeClient: 'url': youtube_url, 'title': title, 'file_path': None, # Will be set when download completes - 'task': None # Will hold the background task } - # Start download in background task - loop = asyncio.get_event_loop() - task = loop.create_task(self._download_internal(download_id, youtube_url, title)) - - # Store task reference - async with self._download_lock: - self.active_downloads[download_id]['task'] = task + # Start download in background thread (returns immediately) + download_thread = threading.Thread( + target=self._download_thread_worker, + args=(download_id, youtube_url, title, filename), + daemon=True + ) + download_thread.start() - logger.info(f"✅ Download started with ID: {download_id}") + logger.info(f"✅ YouTube download {download_id} started in background") return download_id except Exception as e: - logger.error(f"❌ Failed to start download: {e}") + logger.error(f"❌ Failed to start YouTube download: {e}") import traceback traceback.print_exc() return None - async def _download_internal(self, download_id: str, youtube_url: str, title: str): + def _download_thread_worker(self, download_id: str, youtube_url: str, title: str, original_filename: str): """ - Internal method to perform the actual YouTube download in the background. - - Args: - download_id: Unique download ID - youtube_url: YouTube video URL - title: Video title for display + Background thread worker for downloading YouTube videos. + Updates active_downloads dict with progress. """ try: # Update state to downloading - async with self._download_lock: + with self._download_lock: if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Downloading' + self.active_downloads[download_id]['state'] = 'InProgress, Downloading' # Match Soulseek state - # Set current download ID for progress tracking + # Set current download ID for progress hook self.current_download_id = download_id - # Run yt-dlp download in thread pool (to avoid blocking event loop) - loop = asyncio.get_event_loop() + # Perform actual download + file_path = self._download_sync(youtube_url, title) + + # Clear current download ID + self.current_download_id = None + + if file_path: + # Mark as completed/succeeded (match Soulseek state) + with self._download_lock: + if download_id in self.active_downloads: + # IMPORTANT: Keep original filename for context lookup! + # The filename must match what was used to create the context entry + # We store the actual file path separately + self.active_downloads[download_id]['state'] = 'Completed, Succeeded' # Match Soulseek + self.active_downloads[download_id]['progress'] = 100.0 + self.active_downloads[download_id]['file_path'] = file_path + # DO NOT update filename - keep original_filename for context matching + + logger.info(f"✅ YouTube download {download_id} completed: {file_path}") + else: + # Mark as errored + with self._download_lock: + if download_id in self.active_downloads: + self.active_downloads[download_id]['state'] = 'Errored' + + logger.error(f"❌ YouTube download {download_id} failed") + + except Exception as e: + logger.error(f"❌ YouTube download thread failed for {download_id}: {e}") + import traceback + traceback.print_exc() - def _download(): + # Mark as errored + with self._download_lock: + if download_id in self.active_downloads: + self.active_downloads[download_id]['state'] = 'Errored' + + # Clear current download ID + if self.current_download_id == download_id: + self.current_download_id = None + + def _download_sync(self, youtube_url: str, title: str) -> Optional[str]: + """ + Synchronous download method (runs in thread pool executor). + + Args: + youtube_url: YouTube video URL + title: Video title for display + + Returns: + File path if successful, None otherwise + """ + try: + max_retries = 2 + for attempt in range(max_retries): try: # Use default download options download_opts = self.download_opts.copy() + # On retry, try different player client + if attempt > 0: + logger.info(f"🔄 Retry {attempt + 1}/{max_retries} with different settings") + download_opts['extractor_args'] = { + 'youtube': { + 'player_client': ['web'], # Try web-only on retry + 'skip': ['hls', 'dash'], + } + } + # Perform download with yt_dlp.YoutubeDL(download_opts) as ydl: info = ydl.extract_info(youtube_url, download=True) @@ -812,57 +907,38 @@ class YouTubeClient: return str(filename) else: logger.error(f"❌ Download completed but file not found: {filename}") + if attempt < max_retries - 1: + continue # Retry return None except Exception as e: - logger.error(f"❌ Download failed in thread: {e}") - import traceback - traceback.print_exc() - return None - - # Run download - file_path = await loop.run_in_executor(None, _download) - - if file_path: - # Mark download as completed - async with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Completed' - self.active_downloads[download_id]['progress'] = 100.0 - self.active_downloads[download_id]['file_path'] = file_path - - logger.info(f"✅ Download {download_id} completed: {file_path}") - else: - # Mark as error - async with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Errored' - - logger.error(f"❌ Download {download_id} failed") + error_msg = str(e) + logger.error(f"❌ Download attempt {attempt + 1} failed: {error_msg}") + + # Check if it's a 403 error + if '403' in error_msg or 'Forbidden' in error_msg: + if attempt < max_retries - 1: + logger.info(f"⏳ Waiting 2 seconds before retry...") + import time + time.sleep(2) + continue # Retry on 403 + + # For other errors or last retry, print traceback and return + if attempt == max_retries - 1: + import traceback + traceback.print_exc() + else: + continue # Retry - except asyncio.CancelledError: - # Download was cancelled - async with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Cancelled' + return None - logger.info(f"⚠️ Download {download_id} cancelled") - raise + return None # All retries failed except Exception as e: - # Download error - async with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Errored' - - logger.error(f"❌ Download {download_id} failed: {e}") + logger.error(f"❌ Download failed: {e}") import traceback traceback.print_exc() - - finally: - # Clear current download ID - if self.current_download_id == download_id: - self.current_download_id = None + return None async def get_all_downloads(self) -> List[DownloadStatus]: """ @@ -873,7 +949,7 @@ class YouTubeClient: """ download_statuses = [] - async with self._download_lock: + with self._download_lock: for download_id, download_info in self.active_downloads.items(): status = DownloadStatus( id=download_info['id'], @@ -900,7 +976,7 @@ class YouTubeClient: Returns: DownloadStatus object or None if not found """ - async with self._download_lock: + with self._download_lock: if download_id not in self.active_downloads: return None @@ -922,6 +998,9 @@ class YouTubeClient: """ Cancel an active download (matches Soulseek interface). + NOTE: YouTube downloads cannot be truly cancelled mid-download, + but we mark them as cancelled for UI consistency. + Args: download_id: Download ID to cancel username: Ignored for YouTube (kept for interface compatibility) @@ -931,26 +1010,19 @@ class YouTubeClient: True if cancelled successfully, False otherwise """ try: - async with self._download_lock: + with self._download_lock: if download_id not in self.active_downloads: logger.warning(f"⚠️ Download {download_id} not found") return False - download_info = self.active_downloads[download_id] - task = download_info.get('task') - - # Cancel the background task if it exists - if task and not task.done(): - task.cancel() - logger.info(f"⚠️ Cancelled download {download_id}") - - # Update state - download_info['state'] = 'Cancelled' + # Update state to cancelled + self.active_downloads[download_id]['state'] = 'Cancelled' + logger.info(f"⚠️ Marked YouTube download {download_id} as cancelled") # Remove from active downloads if requested if remove: del self.active_downloads[download_id] - logger.info(f"🗑️ Removed download {download_id} from queue") + logger.info(f"🗑️ Removed YouTube download {download_id} from queue") return True diff --git a/web_server.py b/web_server.py index d42e2727..716424c3 100644 --- a/web_server.py +++ b/web_server.py @@ -31,6 +31,7 @@ from core.plex_client import PlexClient from core.jellyfin_client import JellyfinClient from core.navidrome_client import NavidromeClient from core.soulseek_client import SoulseekClient +from core.download_orchestrator import DownloadOrchestrator from core.tidal_client import TidalClient # Added import for Tidal from core.matching_engine import MusicMatchingEngine from core.database_update_worker import DatabaseUpdateWorker, DatabaseStatsWorker @@ -106,7 +107,8 @@ try: plex_client = PlexClient() jellyfin_client = JellyfinClient() navidrome_client = NavidromeClient() - soulseek_client = SoulseekClient() + # Use DownloadOrchestrator instead of SoulseekClient directly (routes between Soulseek/YouTube) + soulseek_client = DownloadOrchestrator() tidal_client = TidalClient() matching_engine = MusicMatchingEngine() sync_service = PlaylistSyncService(spotify_client, plex_client, soulseek_client, jellyfin_client, navidrome_client) @@ -268,6 +270,7 @@ def get_cached_transfer_data(): # Cache expired or empty, fetch new data live_transfers_lookup = {} try: + # First, get Soulseek downloads from API transfers_data = asyncio.run(soulseek_client._make_request('GET', 'transfers/downloads')) if transfers_data: all_transfers = [] @@ -282,7 +285,28 @@ def get_cached_transfer_data(): for transfer in all_transfers: key = f"{transfer.get('username')}::{extract_filename(transfer.get('filename', ''))}" live_transfers_lookup[key] = transfer - + + # Also add YouTube downloads (through orchestrator) + try: + all_downloads = asyncio.run(soulseek_client.get_all_downloads()) + for download in all_downloads: + # Only add YouTube downloads (Soulseek ones are already in the lookup) + if download.username == 'youtube': + key = f"{download.username}::{extract_filename(download.filename)}" + # Convert DownloadStatus to transfer dict format + live_transfers_lookup[key] = { + 'id': download.id, + 'filename': download.filename, + 'username': download.username, + 'state': download.state, + 'percentComplete': download.progress, + 'size': download.size, + 'bytesTransferred': download.transferred, + 'averageSpeed': download.speed, + } + except Exception as e: + print(f"⚠️ Could not fetch YouTube downloads: {e}") + # Update cache transfer_data_cache['data'] = live_transfers_lookup transfer_data_cache['last_update'] = current_time @@ -2058,7 +2082,7 @@ def handle_settings(): if 'active_media_server' in new_settings: config_manager.set_active_media_server(new_settings['active_media_server']) - for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'listenbrainz']: + for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'listenbrainz']: if service in new_settings: for key, value in new_settings[service].items(): config_manager.set(f'{service}.{key}', value) @@ -2073,7 +2097,8 @@ def handle_settings(): spotify_client._setup_client() plex_client.server = None jellyfin_client.server = None - soulseek_client._setup_client() + # Reload orchestrator settings (download source mode, hybrid_primary, etc.) + soulseek_client.reload_settings() # FIX: Re-instantiate the global tidal_client to pick up new settings tidal_client = TidalClient() print("✅ Service clients re-initialized with new settings.") @@ -3551,7 +3576,7 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): Robustly finds a completed file on disk, accounting for name variations and unexpected subdirectories. This version uses the superior normalization logic from the GUI's matching_engine.py to ensure consistency. - + First searches in download_dir, then optionally searches in transfer_dir if provided. Returns tuple (file_path, location) where location is 'downloads' or 'transfer'. """ @@ -3560,6 +3585,14 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): from difflib import SequenceMatcher from unidecode import unidecode + # YOUTUBE SUPPORT: Handle encoded filename format "video_id||title" + # Extract just the title part for file matching + if '||' in api_filename: + print(f"🎵 Detected YouTube encoded filename: {api_filename}") + _, title = api_filename.split('||', 1) + api_filename = title # Use just the title for file searching + print(f"🎵 Extracted title for search: {api_filename}") + def normalize_for_finding(text: str) -> str: """A powerful normalization function adapted from matching_engine.py.""" if not text: return "" @@ -6949,7 +6982,7 @@ def _build_final_path_for_track(context, spotify_artist, album_info, file_ext): playlist_folder_mode = track_info.get("_playlist_folder_mode", False) # Extract year from spotify_album for template use (safe for all modes) - year = 'Unknown' + year = '' # Empty string instead of 'Unknown' to avoid "Unknown albumName" spotify_album = context.get("spotify_album", {}) if spotify_album and spotify_album.get('release_date'): release_date = spotify_album['release_date'] @@ -7087,7 +7120,13 @@ def _apply_path_template(template: str, context: dict) -> str: result = result.replace('$album', context.get('album', 'Unknown Album')) result = result.replace('$title', context.get('title', 'Unknown Track')) result = result.replace('$track', f"{context.get('track_number', 1):02d}") - result = result.replace('$year', str(context.get('year', 'Unknown'))) + result = result.replace('$year', str(context.get('year', ''))) # Empty string instead of 'Unknown' + + # Clean up extra spaces that might result from empty variables + import re + result = re.sub(r'\s+', ' ', result) # Multiple spaces to single space + result = re.sub(r'\s*-\s*-\s*', ' - ', result) # Clean up double dashes + result = result.strip() # Remove leading/trailing spaces return result @@ -7476,7 +7515,40 @@ def _post_process_matched_download_with_verification(context_key, context, file_ context['task_id'] = original_task_id if original_batch_id: context['batch_id'] = original_batch_id - + + # Check if simple download handler already completed everything + if context.get('_simple_download_completed'): + print(f"✅ [Verification] Simple download handler already completed - verifying at custom path") + expected_final_path = context.get('_final_path') + + if expected_final_path and os.path.exists(expected_final_path): + print(f"✅ [Verification] File verified at simple download path: {expected_final_path}") + with tasks_lock: + if task_id in download_tasks: + _mark_task_completed(task_id, context.get('track_info')) + print(f"✅ [Verification] Task {task_id} marked as completed (simple download)") + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + print(f"🗑️ [Verification] Cleaned up context after simple download completion: {context_key}") + + _on_download_completed(batch_id, task_id, success=True) + return + else: + print(f"❌ [Verification] Simple download file not found at: {expected_final_path}") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = "Simple download file not found after processing." + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + _on_download_completed(batch_id, task_id, success=False) + return + # CRITICAL VERIFICATION STEP: Verify the final file exists spotify_artist = context.get("spotify_artist") if not spotify_artist: @@ -7702,6 +7774,10 @@ def _post_process_matched_download(context_key, context, file_path): add_activity_item("✅", "Download Complete", f"{album_name}/{filename}", "Now") logger.info(f"✅ Simple download post-processing complete: {album_name}/{filename}") + + # Set flag in context so verification function knows this was fully handled + context['_simple_download_completed'] = True + context['_final_path'] = str(destination) return # --- END SIMPLE DOWNLOAD HANDLING --- @@ -7756,7 +7832,7 @@ def _post_process_matched_download(context_key, context, file_path): is_album_download = context.get("is_album_download", False) has_clean_spotify_data = context.get("has_clean_spotify_data", False) - + if is_album_download and has_clean_spotify_data: # Build album_info directly from clean Spotify metadata (GUI PARITY) print("✅ Album context with clean Spotify data found - using direct album info") @@ -10383,19 +10459,26 @@ def get_valid_candidates(results, spotify_track, query): if not initial_candidates: return [] - # Filter by user's quality profile before artist verification - # Use shared soulseek_client method for consistency - from core.soulseek_client import SoulseekClient - temp_client = SoulseekClient() - quality_filtered_candidates = temp_client.filter_results_by_quality_preference(initial_candidates) - - # IMPORTANT: Respect empty results from quality filter - # If user has strict quality requirements (e.g., FLAC-only with fallback disabled), - # and no results match, we should fail the download rather than force a fallback. - # The quality filter already has its own fallback logic controlled by the user's settings. - if not quality_filtered_candidates: - print(f"⚠️ [Quality Filter] No candidates match quality profile - download will fail per user preferences") - return [] + # Skip quality filtering for YouTube results (always MP3 320kbps - no quality options) + is_youtube_source = initial_candidates[0].username == "youtube" if initial_candidates else False + + if is_youtube_source: + print(f"🎵 [YouTube] Skipping quality filter - YouTube always provides MP3 320kbps") + quality_filtered_candidates = initial_candidates + else: + # Filter by user's quality profile before artist verification (Soulseek only) + # Use shared soulseek_client method for consistency + from core.soulseek_client import SoulseekClient + temp_client = SoulseekClient() + quality_filtered_candidates = temp_client.filter_results_by_quality_preference(initial_candidates) + + # IMPORTANT: Respect empty results from quality filter + # If user has strict quality requirements (e.g., FLAC-only with fallback disabled), + # and no results match, we should fail the download rather than force a fallback. + # The quality filter already has its own fallback logic controlled by the user's settings. + if not quality_filtered_candidates: + print(f"⚠️ [Quality Filter] No candidates match quality profile - download will fail per user preferences") + return [] verified_candidates = [] spotify_artist_name = spotify_track.artists[0] if spotify_track.artists else "" diff --git a/webui/index.html b/webui/index.html index ab90aabf..7ed669cd 100644 --- a/webui/index.html +++ b/webui/index.html @@ -2547,6 +2547,40 @@ +
+ + +
+ Choose where to download music from. Hybrid mode tries primary source first, then falls back to secondary if it fails. +
+
+ + + +